Skip to content

Commit

Permalink
Use new offline db filename and location
Browse files Browse the repository at this point in the history
  • Loading branch information
gandarez committed Jul 9, 2024
1 parent ccfd8c6 commit b6bff11
Show file tree
Hide file tree
Showing 19 changed files with 786 additions and 101 deletions.
2 changes: 1 addition & 1 deletion USAGE.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ some/submodule/name = new project name
| status_bar_enabled | Turns on wakatime status bar for certain editors. | _bool_ | `true` |
| status_bar_coding_activity | Enables displaying Today's code stats in the status bar of some editors. When false, only the WakaTime icon is displayed in the status bar. | _bool_ | `true` |
| status_bar_hide_categories | When `true`, --today only displays the total code stats, never displaying Categories in the output. | _bool_ | `false` |
| offline | Enables saving code stats locally to ~/.wakatime.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
| offline | Enables saving code stats locally to ~/.wakatime/offline_heartbeats.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
| proxy | Optional proxy configuration. Supports HTTPS, SOCKS and NTLM proxies. For ex: `https://user:pass@host:port`, `socks5://user:pass@host:port`, `domain\\user:pass` | _string_ | |
| no_ssl_verify | Disables SSL certificate verification for HTTPS requests. By default, SSL certificates are verified. | _bool_ | `false` |
| ssl_certs_file | Path to a CA certs file. By default, uses bundled Letsencrypt CA cert along with system ca certs. | _filepath_ | |
Expand Down
54 changes: 53 additions & 1 deletion cmd/offlinesync/offlinesync.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package offlinesync

import (
"fmt"
"os"

cmdapi "github.com/wakatime/wakatime-cli/cmd/api"
"github.com/wakatime/wakatime-cli/cmd/params"
Expand All @@ -25,8 +26,16 @@ func Run(v *viper.Viper) (int, error) {
)
}

err = SyncOfflineActivity(v, queueFilepath)
queueFilepathLegacy, err := offline.QueueFilepathLegacy()

Check warning on line 29 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L29

Added line #L29 was not covered by tests
if err != nil {
log.Warnf("legacy offline sync failed: failed to load offline queue filepath: %s", err)
}

Check warning on line 32 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L31-L32

Added lines #L31 - L32 were not covered by tests

if err = syncOfflineActivityLegacy(v, queueFilepathLegacy); err != nil {
log.Warnf("legacy offline sync failed: %s", err)
}

Check warning on line 36 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L34-L36

Added lines #L34 - L36 were not covered by tests

if err = SyncOfflineActivity(v, queueFilepath); err != nil {

Check warning on line 38 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L38

Added line #L38 was not covered by tests
if errwaka, ok := err.(wakaerror.Error); ok {
return errwaka.ExitCode(), fmt.Errorf("offline sync failed: %s", errwaka.Message())
}
Expand All @@ -42,6 +51,49 @@ func Run(v *viper.Viper) (int, error) {
return exitcode.Success, nil
}

// syncOfflineActivityLegacy syncs the old offline activity by sending heartbeats
// from the legacy offline queue to the WakaTime API.
func syncOfflineActivityLegacy(v *viper.Viper, queueFilepath string) error {
if queueFilepath == "" {
return nil
}

Check warning on line 59 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L58-L59

Added lines #L58 - L59 were not covered by tests

paramOffline := params.LoadOfflineParams(v)

paramAPI, err := params.LoadAPIParams(v)
if err != nil {
return fmt.Errorf("failed to load API parameters: %w", err)
}

Check warning on line 66 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L65-L66

Added lines #L65 - L66 were not covered by tests

apiClient, err := cmdapi.NewClientWithoutAuth(paramAPI)
if err != nil {
return fmt.Errorf("failed to initialize api client: %w", err)
}

Check warning on line 71 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L70-L71

Added lines #L70 - L71 were not covered by tests

if paramOffline.QueueFileLegacy != "" {
queueFilepath = paramOffline.QueueFileLegacy
}

Check warning on line 75 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L74-L75

Added lines #L74 - L75 were not covered by tests

handle := heartbeat.NewHandle(apiClient,
offline.WithSync(queueFilepath, paramOffline.SyncMax),
apikey.WithReplacing(apikey.Config{
DefaultAPIKey: paramAPI.Key,
MapPatterns: paramAPI.KeyPatterns,
}),
)

_, err = handle(nil)
if err != nil {
return err
}

Check warning on line 88 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L87-L88

Added lines #L87 - L88 were not covered by tests

if err := os.Remove(queueFilepath); err != nil {
log.Errorf("failed to delete legacy offline file: %s", err)
}

Check warning on line 92 in cmd/offlinesync/offlinesync.go

View check run for this annotation

Codecov / codecov/patch

cmd/offlinesync/offlinesync.go#L91-L92

Added lines #L91 - L92 were not covered by tests

return nil
}

// SyncOfflineActivity syncs offline activity by sending heartbeats
// from the offline queue to the WakaTime API.
func SyncOfflineActivity(v *viper.Viper, queueFilepath string) error {
Expand Down
144 changes: 144 additions & 0 deletions cmd/offlinesync/offlinesync_internal_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package offlinesync

import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"

"github.com/spf13/viper"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
)

func TestSyncOfflineActivityLegacy(t *testing.T) {
testServerURL, router, tearDown := setupTestServer()
defer tearDown()

var (
plugin = "plugin/0.0.1"
numCalls int
)

router.HandleFunc("/users/current/heartbeats.bulk", func(w http.ResponseWriter, req *http.Request) {
numCalls++

// check request
assert.Equal(t, http.MethodPost, req.Method)
assert.Equal(t, []string{"application/json"}, req.Header["Accept"])
assert.Equal(t, []string{"application/json"}, req.Header["Content-Type"])
assert.Equal(t, []string{"Basic MDAwMDAwMDAtMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDAw"}, req.Header["Authorization"])
assert.True(t, strings.HasSuffix(req.Header["User-Agent"][0], plugin), fmt.Sprintf(
"%q should have suffix %q",
req.Header["User-Agent"][0],
plugin,
))

expectedBody, err := os.ReadFile("testdata/api_heartbeats_request_template.json")
require.NoError(t, err)

body, err := io.ReadAll(req.Body)
require.NoError(t, err)

assert.JSONEq(t, string(expectedBody), string(body))

// send response
w.WriteHeader(http.StatusCreated)

f, err := os.Open("testdata/api_heartbeats_response.json")
require.NoError(t, err)
defer f.Close()

_, err = io.Copy(w, f)
require.NoError(t, err)
})

// setup offline queue
f, err := os.CreateTemp(t.TempDir(), "")
require.NoError(t, err)

db, err := bolt.Open(f.Name(), 0600, nil)
require.NoError(t, err)

dataGo, err := os.ReadFile("testdata/heartbeat_go.json")
require.NoError(t, err)

dataPy, err := os.ReadFile("testdata/heartbeat_py.json")
require.NoError(t, err)

dataJs, err := os.ReadFile("testdata/heartbeat_js.json")
require.NoError(t, err)

insertHeartbeatRecords(t, db, "heartbeats", []heartbeatRecord{
{
ID: "1592868367.219124-file-coding-wakatime-cli-heartbeat-/tmp/main.go-true",
Heartbeat: string(dataGo),
},
{
ID: "1592868386.079084-file-debugging-wakatime-summary-/tmp/main.py-false",
Heartbeat: string(dataPy),
},
{
ID: "1592868394.084354-file-building-wakatime-todaygoal-/tmp/main.js-false",
Heartbeat: string(dataJs),
},
})

err = db.Close()
require.NoError(t, err)

v := viper.New()
v.Set("api-url", testServerURL)
v.Set("key", "00000000-0000-4000-8000-000000000000")
v.Set("sync-offline-activity", 100)
v.Set("plugin", plugin)

err = syncOfflineActivityLegacy(v, f.Name())
require.NoError(t, err)

assert.NoFileExists(t, f.Name())

assert.Eventually(t, func() bool { return numCalls == 1 }, time.Second, 50*time.Millisecond)
}

func setupTestServer() (string, *http.ServeMux, func()) {
router := http.NewServeMux()
srv := httptest.NewServer(router)

return srv.URL, router, func() { srv.Close() }
}

type heartbeatRecord struct {
ID string
Heartbeat string
}

func insertHeartbeatRecords(t *testing.T, db *bolt.DB, bucket string, hh []heartbeatRecord) {
for _, h := range hh {
insertHeartbeatRecord(t, db, bucket, h)
}
}

func insertHeartbeatRecord(t *testing.T, db *bolt.DB, bucket string, h heartbeatRecord) {
t.Helper()

err := db.Update(func(tx *bolt.Tx) error {
b, err := tx.CreateBucketIfNotExists([]byte(bucket))
if err != nil {
return fmt.Errorf("failed to create bucket: %s", err)
}

err = b.Put([]byte(h.ID), []byte(h.Heartbeat))
if err != nil {
return fmt.Errorf("failed put heartbeat: %s", err)
}

return nil
})
require.NoError(t, err)
}
21 changes: 12 additions & 9 deletions cmd/params/params.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"os"
"os/exec"
Expand Down Expand Up @@ -129,10 +130,11 @@ type (

// Offline contains offline related parameters.
Offline struct {
Disabled bool
QueueFile string
PrintMax int
SyncMax int
Disabled bool
QueueFile string
QueueFileLegacy string
PrintMax int
SyncMax int
}

// ProjectParams params for project name sanitization.
Expand Down Expand Up @@ -653,10 +655,11 @@ func LoadOfflineParams(v *viper.Viper) Offline {
}

return Offline{
Disabled: disabled,
QueueFile: vipertools.GetString(v, "offline-queue-file"),
PrintMax: v.GetInt("print-offline-heartbeats"),
SyncMax: syncMax,
Disabled: disabled,
QueueFile: vipertools.GetString(v, "offline-queue-file"),
QueueFileLegacy: vipertools.GetString(v, "offline-queue-file-legacy"),
PrintMax: v.GetInt("print-offline-heartbeats"),
SyncMax: syncMax,
}
}

Expand Down Expand Up @@ -730,7 +733,7 @@ func readExtraHeartbeats() ([]heartbeat.Heartbeat, error) {
in := bufio.NewReader(os.Stdin)

input, err := in.ReadString('\n')
if err != nil {
if err != nil && err != io.EOF {
log.Debugf("failed to read data from stdin: %s", err)
}

Expand Down
9 changes: 9 additions & 0 deletions cmd/params/params_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1687,6 +1687,15 @@ func TestLoad_OfflineQueueFile(t *testing.T) {
assert.Equal(t, "/path/to/file", params.QueueFile)
}

func TestLoad_OfflineQueueFileLegacy(t *testing.T) {
v := viper.New()
v.Set("offline-queue-file-legacy", "/path/to/file")

params := paramscmd.LoadOfflineParams(v)

assert.Equal(t, "/path/to/file", params.QueueFileLegacy)
}

func TestLoad_OfflineSyncMax(t *testing.T) {
v := viper.New()
v.Set("sync-offline-activity", 42)
Expand Down
8 changes: 7 additions & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,11 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {
"",
"(internal) Specify an offline queue file, which will be used instead of the default one.",
)
flags.String(
"offline-queue-file-legacy",
"",
"(internal) Specify the legacy offline queue file, which will be used instead of the default one.",
)

Check warning on line 188 in cmd/root.go

View check run for this annotation

Codecov / codecov/patch

cmd/root.go#L184-L188

Added lines #L184 - L188 were not covered by tests
flags.String(
"output",
"",
Expand Down Expand Up @@ -216,7 +221,7 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {
flags.Int(
"sync-offline-activity",
offline.SyncMaxDefault,
fmt.Sprintf("Amount of offline activity to sync from your local ~/.wakatime.bdb bolt"+
fmt.Sprintf("Amount of offline activity to sync from your local ~/.wakatime/offline_heartbeats.bdb bolt"+

Check warning on line 224 in cmd/root.go

View check run for this annotation

Codecov / codecov/patch

cmd/root.go#L224

Added line #L224 was not covered by tests
" file to your WakaTime Dashboard before exiting. Can be zero or"+
" a positive integer. Defaults to %d, meaning after sending a heartbeat"+
" while online, all queued offline heartbeats are sent to WakaTime API, up"+
Expand Down Expand Up @@ -259,6 +264,7 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {

// hide internal flags
_ = flags.MarkHidden("offline-queue-file")
_ = flags.MarkHidden("offline-queue-file-legacy")

Check warning on line 267 in cmd/root.go

View check run for this annotation

Codecov / codecov/patch

cmd/root.go#L267

Added line #L267 was not covered by tests
_ = flags.MarkHidden("user-agent")

err := v.BindPFlags(flags)
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/wakatime/wakatime-cli

go 1.22.4
go 1.22.5

require (
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
Expand Down
11 changes: 1 addition & 10 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -296,8 +296,6 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -370,8 +368,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand Down Expand Up @@ -435,16 +431,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA=
golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
Expand All @@ -454,8 +447,6 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down
Loading

0 comments on commit b6bff11

Please sign in to comment.