diff --git a/CHANGELOG.md b/CHANGELOG.md
index cff94fe55d7..843a193c551 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -15,6 +15,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Add `"go.opentelemetry.io/contrib/config"` package that includes configuration models generated via go-jsonschema. (#4376)
 - Add `NewSDK` function to `"go.opentelemetry.io/contrib/config"`. The initial implementation only returns noop providers. (#4414)
 - Add metrics support (No-op, OTLP and Prometheus) to `go.opentelemetry.io/contrib/exporters/autoexport`. (#4229, #4479)
+- otelhttp: set unit on the server latency histogram (#4500)
 
 ### Changed
 
diff --git a/instrumentation/net/http/otelhttp/common.go b/instrumentation/net/http/otelhttp/common.go
index 303e5505e41..4fa950dfe2e 100644
--- a/instrumentation/net/http/otelhttp/common.go
+++ b/instrumentation/net/http/otelhttp/common.go
@@ -34,7 +34,7 @@ const (
 	RequestCount          = "http.server.request_count"           // Incoming request count total
 	RequestContentLength  = "http.server.request_content_length"  // Incoming request bytes total
 	ResponseContentLength = "http.server.response_content_length" // Incoming response bytes total
-	ServerLatency         = "http.server.duration"                // Incoming end to end duration, microseconds
+	ServerLatency         = "http.server.duration"                // Incoming end to end duration, milliseconds
 )
 
 // Filter is a predicate used to determine whether a given http.request should
diff --git a/instrumentation/net/http/otelhttp/handler.go b/instrumentation/net/http/otelhttp/handler.go
index b2fbe07841c..a174c005037 100644
--- a/instrumentation/net/http/otelhttp/handler.go
+++ b/instrumentation/net/http/otelhttp/handler.go
@@ -113,7 +113,7 @@ func (h *middleware) createMeasures() {
 	responseBytesCounter, err := h.meter.Int64Counter(ResponseContentLength)
 	handleErr(err)
 
-	serverLatencyMeasure, err := h.meter.Float64Histogram(ServerLatency)
+	serverLatencyMeasure, err := h.meter.Float64Histogram(ServerLatency, metric.WithUnit("ms"))
 	handleErr(err)
 
 	h.counters[RequestContentLength] = requestBytesCounter