Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Merged by Bors] - public metrics #4723

Closed
wants to merge 8 commits into from
Closed

[Merged by Bors] - public metrics #4723

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ed928e6
potential v1.0.3
dshulyak Jul 14, 2023
f87af8d
remove debug test
dshulyak Jul 17, 2023
2e2d308
Merge branch 'potenntial-v1.0.3' into public-metrics
dshulyak Jul 18, 2023
99fa4a2
adjust metrics
dshulyak Jul 18, 2023
002931c
lowercase
dshulyak Jul 18, 2023
2cc3533
add more metrics
dshulyak Jul 19, 2023
2b8354a
more metrics
dshulyak Jul 19, 2023
2b13b04
remove bootnnodes
dshulyak Jul 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions activation/activation.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/spacemeshos/go-spacemesh/datastore"
"github.com/spacemeshos/go-spacemesh/events"
"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/metrics/public"
"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
"github.com/spacemeshos/go-spacemesh/signing"
"github.com/spacemeshos/go-spacemesh/sql"
Expand Down Expand Up @@ -301,6 +302,7 @@ func (b *Builder) generateInitialPost(ctx context.Context) error {
}
events.EmitPostComplete(shared.ZeroChallenge)
metrics.PostDuration.Set(float64(time.Since(startTime).Nanoseconds()))
public.PostSeconds.Set(float64(time.Since(startTime)))
b.log.Info("created the initial post")
if b.verifyInitialPost(ctx, post, metadata) != nil {
return err
Expand Down
3 changes: 2 additions & 1 deletion activation/nipost.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/spacemeshos/go-spacemesh/common/types"
"github.com/spacemeshos/go-spacemesh/events"
"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/metrics/public"
"github.com/spacemeshos/go-spacemesh/signing"
)

Expand Down Expand Up @@ -253,7 +254,7 @@ func (nb *NIPostBuilder) BuildNIPost(ctx context.Context, challenge *types.NIPos
events.EmitPostComplete(nb.state.PoetProofRef[:])
postGenDuration = time.Since(startTime)
nb.log.With().Info("finished post execution", log.Duration("duration", postGenDuration))

public.PostSeconds.Set(postGenDuration.Seconds())
nb.state.NIPost.Post = proof
nb.state.NIPost.PostMetadata = proofMetadata

Expand Down
3 changes: 3 additions & 0 deletions activation/post.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/spacemeshos/go-spacemesh/datastore"
"github.com/spacemeshos/go-spacemesh/events"
"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/metrics/public"
"github.com/spacemeshos/go-spacemesh/sql"
"github.com/spacemeshos/go-spacemesh/sql/atxs"
)
Expand Down Expand Up @@ -315,6 +316,7 @@ func (mgr *PostSetupManager) StartSession(ctx context.Context) error {
log.String("labels_per_unit", fmt.Sprintf("%d", mgr.cfg.LabelsPerUnit)),
log.String("provider", fmt.Sprintf("%d", mgr.lastOpts.ProviderID)),
)
public.InitStart.Set(float64(mgr.lastOpts.NumUnits))
events.EmitInitStart(mgr.id, mgr.commitmentAtxId)
err = mgr.init.Initialize(ctx)

Expand All @@ -337,6 +339,7 @@ func (mgr *PostSetupManager) StartSession(ctx context.Context) error {
events.EmitInitFailure(mgr.id, mgr.commitmentAtxId, err)
return err
}
public.InitEnd.Set(float64(mgr.lastOpts.NumUnits))
events.EmitInitComplete()

mgr.logger.With().Info("post setup completed",
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func AddCommands(cmd *cobra.Command) {
cfg.MetricsPort, "metric server port")
cmd.PersistentFlags().StringVar(&cfg.MetricsPush, "metrics-push",
cfg.MetricsPush, "Push metrics to url")
cmd.PersistentFlags().IntVar(&cfg.MetricsPushPeriod, "metrics-push-period",
cmd.PersistentFlags().DurationVar(&cfg.MetricsPushPeriod, "metrics-push-period",
cfg.MetricsPushPeriod, "Push period")
cmd.PersistentFlags().StringArrayVar(&cfg.PoETServers, "poet-server",
cfg.PoETServers, "The poet server url. (temporary) Can be passed multiple times")
Expand Down
9 changes: 5 additions & 4 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,11 @@ type BaseConfig struct {
CollectMetrics bool `mapstructure:"metrics"`
MetricsPort int `mapstructure:"metrics-port"`

MetricsPush string `mapstructure:"metrics-push"`
MetricsPushPeriod int `mapstructure:"metrics-push-period"`
MetricsPushUser string `mapstructure:"metrics-push-user"`
MetricsPushPass string `mapstructure:"metrics-push-pass"`
MetricsPush string `mapstructure:"metrics-push"`
MetricsPushPeriod time.Duration `mapstructure:"metrics-push-period"`
MetricsPushUser string `mapstructure:"metrics-push-user"`
MetricsPushPass string `mapstructure:"metrics-push-pass"`
MetricsPushHeader map[string]string `mapstructure:"metrics-push-header"`

ProfilerName string `mapstructure:"profiler-name"`
ProfilerURL string `mapstructure:"profiler-url"`
Expand Down
12 changes: 10 additions & 2 deletions config/mainnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func MainnetConfig() Config {
if err := postPowDifficulty.UnmarshalText([]byte("00037ec8ec25e6d2c00000000000000000000000000000000000000000000000")); err != nil {
panic(err)
}

p2pconfig := p2p.DefaultConfig()
return Config{
BaseConfig: BaseConfig{
DataDirParent: defaultDataDir,
Expand All @@ -45,9 +45,17 @@ func MainnetConfig() Config {
OptFilterThreshold: 90,

TickSize: 9331200,
PoETServers: []string{
"https://mainnet-poet-0.spacemesh.network",
"https://mainnet-poet-1.spacemesh.network",
"https://mainnet-poet-2.spacemesh.network",
"https://poet-110.spacemesh.network",
"https://poet-111.spacemesh.network",
},
},
Genesis: &GenesisConfig{
GenesisTime: "2023-07-14T08:00:00Z",
ExtraData: "00000000000000000001a6bc150307b5c1998045752b3c87eccf3c013036f3cc",
Accounts: MainnetAccounts(),
},
Tortoise: tortoise.Config{
Expand Down Expand Up @@ -105,7 +113,7 @@ func MainnetConfig() Config {
DataDir: os.TempDir(),
Interval: 30 * time.Second,
},
P2P: p2p.DefaultConfig(),
P2P: p2pconfig,
API: grpcserver.DefaultConfig(),
TIME: timeConfig.DefaultConfig(),
SMESHING: DefaultSmeshingConfig(),
Expand Down
29 changes: 29 additions & 0 deletions metrics/public/public.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package public

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

var Registry = prometheus.NewRegistry()

var (
Connections = promauto.With(Registry).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "smh",
Name: "connections",
}, []string{"dir"})
initSize = promauto.With(Registry).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "smh",
Name: "init_size",
}, []string{"step"})
InitStart = initSize.WithLabelValues("start")
InitEnd = initSize.WithLabelValues("complete")
PostSeconds = promauto.With(Registry).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "smh",
Name: "post_seconds",
}, []string{}).WithLabelValues()
Version = promauto.With(Registry).NewGaugeVec(prometheus.GaugeOpts{
Namespace: "smh",
Name: "version",
}, []string{"version"})
)
24 changes: 13 additions & 11 deletions metrics/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,31 @@
package metrics

import (
"net/http"
"time"

stdprometheus "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/push"

"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/metrics/public"
)

// StartPushingMetrics begins pushing metrics to the url specified by the --metrics-push flag
// with period specified by the --metrics-push-period flag.
func StartPushingMetrics(url, username, password string, periodSec int, nodeID, networkID string) {
period := time.Duration(periodSec) * time.Second
ticker := time.NewTicker(period)

pusher := push.New(url, "go-spacemesh").Gatherer(stdprometheus.DefaultGatherer).
Grouping("node_id", nodeID).
Grouping("network_id", networkID)

func StartPushingMetrics(url, username, password string, headers map[string]string, period time.Duration, nodeID, networkID string) {
header := http.Header{}
for k, v := range headers {
header.Add(k, v)
}
pusher := push.New(url, "go-spacemesh").Gatherer(public.Registry).
Grouping("node", nodeID).
Grouping("network", networkID).
Header(header)

Check warning on line 23 in metrics/push.go

View check run for this annotation

Codecov / codecov/patch

metrics/push.go#L15-L23 

Added lines #L15 - L23 were not covered by tests
if username != "" && password != "" {
pusher.BasicAuth(username, password)
pusher = pusher.BasicAuth(username, password)

Check warning on line 25 in metrics/push.go

View check run for this annotation

Codecov / codecov/patch

metrics/push.go#L25 

Added line #L25 was not covered by tests
}

go func() {
ticker := time.NewTicker(period)

Check warning on line 28 in metrics/push.go

View check run for this annotation

Codecov / codecov/patch

metrics/push.go#L28 

Added line #L28 was not covered by tests
for range ticker.C {
err := pusher.Push()
if err != nil {
Expand Down
13 changes: 9 additions & 4 deletions node/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
"github.com/spacemeshos/go-spacemesh/malfeasance"
"github.com/spacemeshos/go-spacemesh/mesh"
"github.com/spacemeshos/go-spacemesh/metrics"
"github.com/spacemeshos/go-spacemesh/metrics/public"
"github.com/spacemeshos/go-spacemesh/miner"
"github.com/spacemeshos/go-spacemesh/node/mapstructureutil"
"github.com/spacemeshos/go-spacemesh/p2p"
Expand Down Expand Up @@ -453,9 +454,9 @@
timeCfg.TimeConfigValues = app.Config.TIME

app.setupLogging()

app.introduction()

public.Version.WithLabelValues(cmd.Version).Set(1)
return nil
}

Expand Down Expand Up @@ -1374,9 +1375,13 @@
}

if app.Config.MetricsPush != "" {
metrics.StartPushingMetrics(app.Config.MetricsPush,
app.Config.MetricsPushUser, app.Config.MetricsPushPass, app.Config.MetricsPushPeriod,
app.host.ID().String(), app.Config.Genesis.GenesisID().ShortString())
metrics.StartPushingMetrics(
app.Config.MetricsPush,
app.Config.MetricsPushUser,
app.Config.MetricsPushPass,
app.Config.MetricsPushHeader,
app.Config.MetricsPushPeriod,
app.host.ID().String()[:5], app.Config.Genesis.GenesisID().ShortString())

Check warning on line 1384 in node/node.go

View check run for this annotation

Codecov / codecov/patch

node/node.go#L1378-L1384 

Added lines #L1378 - L1384 were not covered by tests
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you think about hashing the first 5 char?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe hash all then and use 5 chars? i can do

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually it doesn't hide anything, if i want i can hash all identities found in the network and match them

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes. but at least its a step towards anomynizing. somebody have to pay the computation to get it....

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the same info will be publicly available in the epoch when ATX will be published, right? So why are we trying to hide it?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is p2p id

}

if err := app.startServices(ctx); err != nil {
Expand Down
8 changes: 6 additions & 2 deletions p2p/metrics/connections.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package metrics

import (
"strings"
"time"

"github.com/libp2p/go-libp2p/core/network"
ma "github.com/multiformats/go-multiaddr"

"github.com/spacemeshos/go-spacemesh/metrics"
"github.com/spacemeshos/go-spacemesh/metrics/public"
)

var (
Expand Down Expand Up @@ -52,12 +54,14 @@ func (c *ConnectionsMeeter) Listen(network.Network, ma.Multiaddr) {}
func (c *ConnectionsMeeter) ListenClose(network.Network, ma.Multiaddr) {}

// Connected called when a connection opened.
func (c *ConnectionsMeeter) Connected(network.Network, network.Conn) {
func (c *ConnectionsMeeter) Connected(_ network.Network, conn network.Conn) {
public.Connections.WithLabelValues(strings.ToLower(conn.Stat().Direction.String())).Inc()
connections.WithLabelValues().Inc()
}

// Disconnected called when a connection closed.
func (c *ConnectionsMeeter) Disconnected(network.Network, network.Conn) {
func (c *ConnectionsMeeter) Disconnected(_ network.Network, conn network.Conn) {
public.Connections.WithLabelValues(strings.ToLower(conn.Stat().Direction.String())).Dec()
connections.WithLabelValues().Dec()
}

Expand Down