From ce8ebb793f259105b3511510d18e99ddcbb0a814 Mon Sep 17 00:00:00 2001 From: borg323 Date: Fri, 13 Dec 2019 12:34:19 +0200 Subject: [PATCH 01/12] remove hack for fp16 detection under windows --- lc0_main.go | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index 488c8e6..28c76d3 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -43,7 +43,6 @@ var ( hasCudnnFp16 bool hasOpenCL bool hasBlas bool - testedCudnnFp16 bool hostname = flag.String("hostname", "http://api.lczero.org", "Address of the server") user = flag.String("user", "", "Username") @@ -406,19 +405,12 @@ func (c *cmdWrapper) launch(networkPath string, otherNetPath string, args []stri last_fp_threshold = -1.0 case strings.HasPrefix(line, "bestmove "): // fmt.Println(line) - testedCudnnFp16 = true c.BestMove <- strings.Split(line, " ")[1] case strings.HasPrefix(line, "id name Lc0 "): c.Version = strings.Split(line, " ")[3] fmt.Println(line) case strings.HasPrefix(line, "info"): - testedCudnnFp16 = true - case strings.HasPrefix(line, "GPU compute capability:"): - cc, _ := strconv.ParseFloat(strings.Split(line, " ")[3], 32) - if cc >= 7.0 { - testedCudnnFp16 = true - } - fallthrough + break default: fmt.Println(line) } @@ -564,18 +556,10 @@ func playMatch(httpClient *http.Client, ngr client.NextGameResponse, baselinePat } case gi, ok := <-c.gi: if !ok { - // Under windows we don't get the exception, so also check here. - if hasCudnnFp16 && !testedCudnnFp16 && *backopts == "" { - log.Println("GPU probably doesn't support the cudnn-fp16 backend") - hasCudnnFp16 = false - close(reverseDoneCh) - return nil, errors.New("retry") - } log.Printf("GameInfo channel closed, exiting match loop") done = true break } - testedCudnnFp16 = true progressOrKill = true trainDirHolder[0] = path.Dir(gi.fname) wg.Add(1) @@ -645,17 +629,10 @@ func train(httpClient *http.Client, ngr client.NextGameResponse, } case gi, ok := <-c.gi: if !ok { - // Under windows we don't get the exception, so also check here. - if hasCudnnFp16 && !testedCudnnFp16 && *backopts == "" { - log.Println("GPU probably doesn't support the cudnn-fp16 backend") - hasCudnnFp16 = false - return errors.New("retry") - } log.Printf("GameInfo channel closed, exiting train loop") done = true break } - testedCudnnFp16 = true fmt.Printf("Uploading game: %d\n", numGames) numGames++ progressOrKill = true From b4371bda152779cd2303eebef3fa51896a041aa1 Mon Sep 17 00:00:00 2001 From: borg323 <39573933+borg323@users.noreply.github.com> Date: Wed, 8 Apr 2020 19:47:24 +0300 Subject: [PATCH 02/12] Bump version post release (#115) --- lc0_main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lc0_main.go b/lc0_main.go index 54521a1..5bc7c05 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -133,7 +133,7 @@ func getExtraParams() map[string]string { return map[string]string{ "user": *user, "password": *password, - "version": "26", + "version": "27", "token": strconv.Itoa(randId), "train_only": strconv.FormatBool(*trainOnly), "hostname": *localHost, From cc05a3c0d6ec070c07b6e3f636b4df33d0590634 Mon Sep 17 00:00:00 2001 From: borg323 <39573933+borg323@users.noreply.github.com> Date: Mon, 4 May 2020 09:41:45 +0300 Subject: [PATCH 03/12] Check for eigen instead of blas for validating the dx12 backend (#117) --- lc0_main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index 5bc7c05..0c84717 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -43,7 +43,7 @@ var ( hasCudnn bool hasCudnnFp16 bool hasOpenCL bool - hasBlas bool + hasEigen bool hasDx bool testedCudnnFp16 bool testedDxNet string @@ -293,8 +293,8 @@ func checkLc0() { if err != nil { log.Fatal(err) } - if bytes.Contains(out, []byte("blas")) { - hasBlas = true + if bytes.Contains(out, []byte("eigen")) { + hasEigen = true } if bytes.Contains(out, []byte("dx12")) { hasDx = true @@ -315,7 +315,7 @@ func checkLc0() { func checkDx(networkPath string) { dir, _ := os.Getwd() - if !hasBlas { + if !hasEigen { log.Fatalf("Dx12 backend cannot be validated") } log.Println("Sanity checking the dx12 driver.") From 98a52e86affef9a3259139c13df1d7864c891c07 Mon Sep 17 00:00:00 2001 From: borg323 Date: Mon, 8 Jun 2020 17:53:27 +0300 Subject: [PATCH 04/12] rename executable to lc0-training-client --- appveyor.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 561b9da..bd4fa2c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,21 +7,21 @@ environment: matrix: - NAME: .exe GOOS: windows - - NAME: _linux + - NAME: -linux GOOS: linux - - NAME: _mac + - NAME: -mac GOOS: darwin install: - go get -u github.com/Tilps/chess - go get -u github.com/nightlyone/lockfile build_script: - - go build -o client%NAME% lc0_main.go + - go build -o lc0-training-client%NAME% lc0_main.go artifacts: - - path: client$(NAME) - name: client + - path: lc0-training-client$(NAME) + name: lc0-training-client deploy: - provider: GitHub - artifact: client%NAME% + artifact: lc0-training-client%NAME% auth_token: secure: USFAdwQKTXqOXQjCYQfzWvzRpUhvqJLBkN4hbOg+j876vDxGZHt9bMYayb5evePp on: From 01ba6b58185423a7c8cda1eb6ec7736398752357 Mon Sep 17 00:00:00 2001 From: borg323 Date: Wed, 10 Jun 2020 02:57:13 +0300 Subject: [PATCH 05/12] option and default directory for config file --- lc0_main.go | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index 0c84717..a39245e 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -49,7 +49,6 @@ var ( testedDxNet string lc0Exe = "lc0" - settingsPath = "settings.json" defaultLocalHost = "Unknown" gpuType = "Unknown" @@ -70,6 +69,7 @@ var ( trainOnly = flag.Bool("train-only", false, "Do not play match games") report_host = flag.Bool("report-host", false, "Send hostname to server for more fine-grained statistics") report_gpu = flag.Bool("report-gpu", false, "Send gpu info to server for more fine-grained statistics") + settingsPath = flag.String("config", "", "JSON configuration file to use") ) // Settings holds username and password. @@ -1114,13 +1114,34 @@ func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) - settingsUser, settingsPassword, settingsHost := readSettings(settingsPath) + if runtime.GOOS == "linux" && len(*settingsPath) == 0 { + configDir := os.Getenv("XDG_CONFIG_HOME") + if len(configDir) != 0 { + *settingsPath = configDir + "/lc0-training-client-config.json" + } else { + homeDir := os.Getenv("HOME") + if len(homeDir) != 0 { + *settingsPath = homeDir + "/.config/lc0-training-client-config.json" + } + } + } else if runtime.GOOS == "darwin" && len(*settingsPath) == 0 { + homeDir := os.Getenv("HOME") + if len(homeDir) != 0 { + *settingsPath = homeDir + "/Library/Preferences/lc0-training-client-config.json" + } + } + + if len(*settingsPath) == 0 { + *settingsPath = "lc0-training-client-config.json" + } + + settingsUser, settingsPassword, settingsHost := readSettings(*settingsPath) if len(*user) == 0 || len(*password) == 0 { *user = settingsUser *password = settingsPassword if len(*user) == 0 || len(*password) == 0 { - *user, *password = createSettings(settingsPath) + *user, *password = createSettings(*settingsPath) } } From f2f78bfad1eaea6a986005fcd711972f3311dc53 Mon Sep 17 00:00:00 2001 From: borg323 Date: Thu, 11 Jun 2020 10:04:52 +0300 Subject: [PATCH 06/12] rework logic to use config subdirectory --- lc0_main.go | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index a39245e..4e33003 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -1114,25 +1114,32 @@ func main() { log.SetFlags(log.LstdFlags | log.Lshortfile) - if runtime.GOOS == "linux" && len(*settingsPath) == 0 { - configDir := os.Getenv("XDG_CONFIG_HOME") - if len(configDir) != 0 { - *settingsPath = configDir + "/lc0-training-client-config.json" - } else { + if len(*settingsPath) == 0 { + *settingsPath = "lc0-training-client-config.json" + configDir := "" + if runtime.GOOS == "linux" { + configDir = os.Getenv("XDG_CONFIG_HOME") + if len(configDir) == 0 { + homeDir := os.Getenv("HOME") + if len(homeDir) != 0 { + configDir = homeDir + "/.config" + } + } + } else if runtime.GOOS == "darwin" { homeDir := os.Getenv("HOME") if len(homeDir) != 0 { - *settingsPath = homeDir + "/.config/lc0-training-client-config.json" + configDir = homeDir + "/Library/Preferences" } } - } else if runtime.GOOS == "darwin" && len(*settingsPath) == 0 { - homeDir := os.Getenv("HOME") - if len(homeDir) != 0 { - *settingsPath = homeDir + "/Library/Preferences/lc0-training-client-config.json" - } - } - if len(*settingsPath) == 0 { - *settingsPath = "lc0-training-client-config.json" + if len(configDir) != 0 { + configDir = filepath.Join(configDir, "lc0") + _, err = os.Stat(configDir) + if os.IsNotExist(err) { + err = os.MkdirAll(configDir, os.ModePerm) + } + *settingsPath = filepath.Join(configDir, *settingsPath) + } } settingsUser, settingsPassword, settingsHost := readSettings(*settingsPath) From 3f763906749da59429f493054bcb577a03459a5d Mon Sep 17 00:00:00 2001 From: borg323 Date: Thu, 11 Jun 2020 10:33:04 +0300 Subject: [PATCH 07/12] option and default directory for cache --- lc0_main.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lc0_main.go b/lc0_main.go index 0c84717..0e7ee08 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -63,6 +63,7 @@ var ( backopts = flag.String("backend-opts", "", `Options for the lc0 mux. backend. Example: --backend-opts="cudnn(gpu=1)"`) parallel = flag.Int("parallelism", -1, "Number of games to play in parallel (-1 for default)") + cacheDir = flag.String("cache", "", "Directory to use for downloaded networks cache") useTestServer = flag.Bool("use-test-server", false, "Set host name to test server.") runId = flag.Uint("run", 0, "Which training run to contribute to (default 0 to let server decide)") keep = flag.Bool("keep", false, "Do not delete old network files") @@ -842,6 +843,30 @@ func acquireLock(dir string, sha string) (lockfile.Lockfile, error) { func getNetwork(httpClient *http.Client, sha string, keepTime string) (string, error) { dir := "client-cache" + if len(*cacheDir) != 0 { + dir = *cacheDir + } else { + userCache := "" + if runtime.GOOS == "linux" { + userCache = os.Getenv("XDG_CACHE_HOME") + if len(userCache) == 0 { + homeDir := os.Getenv("HOME") + if len(homeDir) != 0 { + userCache = homeDir + "/.cache" + } + } + } else if runtime.GOOS == "darwin" { + homeDir := os.Getenv("HOME") + if len(homeDir) != 0 { + userCache = homeDir + "/Library/Caches" + } + } + + if len(userCache) != 0 { + userCache = filepath.Join(userCache, "lc0") + dir = filepath.Join(userCache, dir) + } + } os.MkdirAll(dir, os.ModePerm) if keepTime != inf { err := removeAllExcept(dir, sha, keepTime) From 4b43e2def1cc13bf2f556dccfd00851fcb4caec1 Mon Sep 17 00:00:00 2001 From: borg323 Date: Thu, 11 Jun 2020 10:39:46 +0300 Subject: [PATCH 08/12] fix last reference to settings.json --- lc0_main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lc0_main.go b/lc0_main.go index 4e33003..cdf6bce 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -52,7 +52,7 @@ var ( defaultLocalHost = "Unknown" gpuType = "Unknown" - localHost = flag.String("localhost", "", "Localhost name to send to the server when reporting (defaults to Unknown, overridden by settings.json)") + localHost = flag.String("localhost", "", "Localhost name to send to the server when reporting\n(defaults to Unknown, overridden by the configuration file)") hostname = flag.String("hostname", "http://api.lczero.org", "Address of the server") user = flag.String("user", "", "Username") password = flag.String("password", "", "Password") From fbc27d056570a5ec83d588d1528e1b7e6075085a Mon Sep 17 00:00:00 2001 From: borg323 Date: Thu, 11 Jun 2020 12:13:52 +0300 Subject: [PATCH 09/12] also check for lc0 on the path --- lc0_main.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index 0c84717..7490706 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -286,8 +286,7 @@ func createCmdWrapper() *cmdWrapper { } func checkLc0() { - dir, _ := os.Getwd() - cmd := exec.Command(path.Join(dir, lc0Exe)) + cmd := exec.Command(lc0Exe) cmd.Args = append(cmd.Args, "--help") out, err := cmd.CombinedOutput() if err != nil { @@ -314,12 +313,11 @@ func checkLc0() { } func checkDx(networkPath string) { - dir, _ := os.Getwd() if !hasEigen { log.Fatalf("Dx12 backend cannot be validated") } log.Println("Sanity checking the dx12 driver.") - cmd := exec.Command(path.Join(dir, lc0Exe)) + cmd := exec.Command(lc0Exe) sGpu := "" if *gpu >= 0 { sGpu = fmt.Sprintf(",gpu=%v", *gpu) @@ -339,8 +337,7 @@ func checkDx(networkPath string) { } func (c *cmdWrapper) launch(networkPath string, otherNetPath string, args []string, input bool) { - dir, _ := os.Getwd() - c.Cmd = exec.Command(path.Join(dir, lc0Exe)) + c.Cmd = exec.Command(lc0Exe) // Add the "selfplay" or "uci" part first mode := args[0] c.Cmd.Args = append(c.Cmd.Args, mode) @@ -1091,7 +1088,11 @@ func main() { if runtime.GOOS == "windows" { lc0Exe = "lc0.exe" } - + dir, _ := os.Getwd() + fi, err := os.Stat(path.Join(dir, lc0Exe)) + if err == nil && !fi.Mode().Perm().IsDir() { + lc0Exe = path.Join(dir, lc0Exe) + } checkLc0() maybeSetTrainOnly() @@ -1101,7 +1102,7 @@ func main() { log.Fatal("Training run number too large") } randBytes := make([]byte, 2) - _, err := rand.Reader.Read(randBytes) + _, err = rand.Reader.Read(randBytes) if err != nil { randId = -1 } else { From 6092a988902b55f8c3ff662145ac7bbce64f2cca Mon Sep 17 00:00:00 2001 From: borg323 Date: Mon, 29 Jun 2020 03:32:55 +0300 Subject: [PATCH 10/12] create only the lc0 config directory --- lc0_main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index cdf6bce..95a4fbf 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -1136,9 +1136,11 @@ func main() { configDir = filepath.Join(configDir, "lc0") _, err = os.Stat(configDir) if os.IsNotExist(err) { - err = os.MkdirAll(configDir, os.ModePerm) + err = os.Mkdir(configDir, os.ModePerm) + } + if err == nil { + *settingsPath = filepath.Join(configDir, *settingsPath) } - *settingsPath = filepath.Join(configDir, *settingsPath) } } From 9ec4a1dcd033a354450bc6039dd9f77282743c99 Mon Sep 17 00:00:00 2001 From: borg323 Date: Mon, 29 Jun 2020 03:46:59 +0300 Subject: [PATCH 11/12] verify user cache directory exists --- lc0_main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lc0_main.go b/lc0_main.go index 0e7ee08..ebee8e8 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -863,8 +863,11 @@ func getNetwork(httpClient *http.Client, sha string, keepTime string) (string, e } if len(userCache) != 0 { - userCache = filepath.Join(userCache, "lc0") - dir = filepath.Join(userCache, dir) + _, err := os.Stat(userCache) + if err == nil { + userCache = filepath.Join(userCache, "lc0") + dir = filepath.Join(userCache, dir) + } } } os.MkdirAll(dir, os.ModePerm) From 2ca7edbfe118f506f8390e9c36109500a0084668 Mon Sep 17 00:00:00 2001 From: Tilps Date: Mon, 29 Jun 2020 12:10:32 +1000 Subject: [PATCH 12/12] Bump version post-release branch point. --- lc0_main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lc0_main.go b/lc0_main.go index 13b9e54..b5a73e0 100644 --- a/lc0_main.go +++ b/lc0_main.go @@ -133,7 +133,7 @@ func getExtraParams() map[string]string { return map[string]string{ "user": *user, "password": *password, - "version": "27", + "version": "28", "token": strconv.Itoa(randId), "train_only": strconv.FormatBool(*trainOnly), "hostname": *localHost,