Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bug: steam IO encode performance is unexpect, freeBytes is not done #542

Closed
201430098137 opened this issue Oct 25, 2023 · 3 comments · Fixed by #547
Closed

bug: steam IO encode performance is unexpect, freeBytes is not done #542

201430098137 opened this issue Oct 25, 2023 · 3 comments · Fixed by #547

Comments

@201430098137
Copy link

recently, I use stram IO to reduct memory usage, but memory usage is higher then sonic.Marshal, I bench it.
I encode map[string]interface{} to json and write it to file,

var cases = []map[string]interface{}{
	{
		"dsds": "dsdds",
		"dsalkjlsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
	},
	{
		"dsdsd": "dsdsd",
		"dsalkjalsd":   "sasasas",
		"dsalkjlsdsdd":  "sasasas",
		"dsalkjlsdss":  "sasasas",
		"dsalkjlasdsd":  "sasasas",
		"dsalkjlssfddsf": "sasasas",
		"dsalkjlssdfd":   "sasasas",
		"dsalkjlssdd":  "sasasas",
		"dsalkjlssds":  "sasasas",
		"dsalkjlassd":  "sasasas",
		"dsalkjlscdsf": "sasasas",
		"dsalkjlsdsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
	},
	{
		"dsdsd": "dsdsd",
		"dsalkjalsd":   "sasasas",
		"dsalkjlsdsdd":  "sasasas",
		"dsalkjlsdss":  "sasasas",
		"dsalkjlasdsd":  "sasasas",
		"dsalkjlssfddsf": "sasasas",
		"dsalkjlssdfd":   "sasasas",
		"dsalkjlssdd":  "sasasas",
		"dsalkjlssds":  "sasasas",
		"dsalkjlassd":  "sasasas",
		"dsalkjlscdsf": "sasasas",
		"dsalkjlsdsd":   "sasasas",
		"dsalkjlsdd":  "sasasas",
		"dsalkjlsds":  "sasasas",
		"dsalkjlasd":  "sasasas",
		"dsalkjlsdsf": "sasasas",
		"sasdas":"asa",
		"saasdss":"asa",
		"sassdc":"asa",
		"sasvsdsdas":"asa",
		"sasjyja":"asa",
		"sagncs":"asa",
		"sabdfss":"asa",
		"santys":"asa",
		"saabfdss":"asa",
		"sasbdfc":"asa",
		"sasbdfas":"asa",
		"sabdfsa":"asa",
		"sacdbfas":"asa",
		"sasss":"asa",
		"sasaasf":"asa",
		"saasdass":"asa",
		"sasavsasc":"asa",
		"sfda":"asa",
		"saasfsa":"asa",
		"sasafscs":"asa",
		"asf":"asa",
		"vxxz":"asa",
		"zxc":"asa",
		"nrt":"asa",
		"dbf":"asa",
		"sabwdfsa":"asa",
		"bdf":"asa",
		"sazxvss":"asa",
		"sasvzv":"asa",
		"saaczczxss":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"cz":"asa",
		"sasvzxas":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"vz":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sazxvcs":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sasvzxs":map[string]interface{}{

			"dsdsd": "dsdsd",
			"dsalkjalsd":   "sasasas",
			"dsalkjlsdsdd":  "sasasas",
			"dsalkjlsdss":  "sasasas",
			"dsalkjlasdsd":  "sasasas",
			"dsalkjlssfddsf": "sasasas",
			"dsalkjlssdfd":   "sasasas",
			"dsalkjlssdd":  "sasasas",
			"dsalkjlssds":  "sasasas",
			"dsalkjlassd":  "sasasas",
			"dsalkjlscdsf": "sasasas",
			"dsalkjlsdsd":   "sasasas",
			"dsalkjlsdd":  "sasasas",
			"dsalkjlsds":  "sasasas",
			"dsalkjlasd":  "sasasas",
			"dsalkjlsdsf": "sasasas",
		},
		"sl;kas":map[string]interface{}{

				"dsdsd": "dsdsd",
				"dsalkjalsd":   "sasasas",
				"dsalkjlsdsdd":  "sasasas",
				"dsalkjlsdss":  "sasasas",
				"dsalkjlasdsd":  "sasasas",
				"dsalkjlssfddsf": "sasasas",
				"dsalkjlssdfd":   "sasasas",
				"dsalkjlssdd":  "sasasas",
				"dsalkjlssds":  "sasasas",
				"dsalkjlassd":  "sasasas",
				"dsalkjlscdsf": "sasasas",
				"dsalkjlsdsd":   "sasasas",
				"dsalkjlsdd":  "sasasas",
				"dsalkjlsds":  "sasasas",
				"dsalkjlasd":  "sasasas",
				"dsalkjlsdsf": "sasasas",
		},

	},
}
func BenchmarkEncode(b *testing.B) {
	fd, err := os.OpenFile("/tmp/aa.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
	if err != nil {
		b.Errorf("打开文件错误：%s", err.Error())
	}
	var enc = sonic.ConfigDefault.NewEncoder(fd)
	for i := 0; i < b.N; i++ {
		err := enc.Encode(cases[i%len(cases)])
		if err != nil {
			b.Errorf("enc.Encode错误：%s", err.Error())
		}
	}
}

func BenchmarkMarshal(b *testing.B) {
	fd, err := os.OpenFile("/tmp/bb.json", os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
	if err != nil {
		b.Errorf("打开文件错误：%s", err.Error())
	}

	for i := 0; i < b.N; i++ {
		bytes, err := sonic.Marshal(cases[i%len(cases)])
		if err != nil {
			b.Errorf("sonic.Marshal错误：%s", err.Error())
		}
		_, err = fd.Write(bytes)
		if err != nil {
			b.Errorf("Write错误：%s", err.Error())
		}
		_, err = fd.Write([]byte{'\n'})
		if err != nil {
			b.Errorf("Write错误：%s", err.Error())
		}
	}

}

profile memory
stream IO
image
sonic.Marshal
image

Interestingly, stream IO use more memory then sonic.Marshal.
I debugger that code, I discover the issue.

image
line 71 out is set out[n:], data of out is unaccess, line 81 freeBytes is useless.
image
@201430098137
Copy link
Author

code: github.com/bytedance/sonic@v1.10.2/internal/encoder/stream.go:70

@liuq19
Copy link
Collaborator

liuq19 commented Oct 27, 2023

Thanks. we will investigate it. cc @AsterDY

@AsterDY AsterDY mentioned this issue Oct 31, 2023
Merged
@AsterDY
Copy link
Collaborator

AsterDY commented Oct 31, 2023

fixed by #547

@AsterDY AsterDY closed this as completed Oct 31, 2023
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

fix: didn't fully reuse buffer memory bytedance/sonic
3 participants
@201430098137 @liuq19 @AsterDY