Skip to content

Commit

Permalink
Use base32 instead of base64 for identifiers (#21)
Browse files Browse the repository at this point in the history
Base64 in Go 1.22 requires that the list of characters
in the encoding be unique. This is not possible
if we are using it to generate Go identifiers.

So, we switch to base32.

This is not a breaking change as this only affects
the generated code, and not the wire format.
  • Loading branch information
Christopher Swenson committed Mar 8, 2024
1 parent c1a7c12 commit 1750000
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 73 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@ jobs:
- uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0
with:
go-version: ${{ needs.get-go-version.outputs.go-version }}
- run: go test -v ./codec
- run: go test -v ./codec
- run: go test -tags codecgen.exec -v ./codec
138 changes: 72 additions & 66 deletions codec/doc.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,10 @@ Rich Feature Set includes:
- Drop-in replacement for encoding/json. `json:` key in struct tag supported.
- Provides a RPC Server and Client Codec for net/rpc communication protocol.
- Handle unique idiosyncrasies of codecs e.g.
- For messagepack, configure how ambiguities in handling raw bytes are resolved
- For messagepack, provide rpc server/client codec to support
msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
- For messagepack, configure how ambiguities in handling raw bytes are resolved
- For messagepack, provide rpc server/client codec to support
msgpack-rpc protocol defined at:
https://github.com/msgpack-rpc/msgpack-rpc/blob/master/spec.md
## Extension Support
Expand All @@ -75,19 +74,20 @@ custom types.
There are no restrictions on what the custom type can be. Some examples:
```go
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
type BisSet []int
type BitSet64 uint64
type UUID string
type MyStructWithUnexportedFields struct { a int; b bool; c []int; }
type GifImage struct { ... }
```
As an illustration, MyStructWithUnexportedFields would normally be encoded
as an empty map because it has no exported fields, while UUID would be
encoded as a string. However, with extension support, you can encode any of
these however you like.
## Custom Encoding and Decoding
This package maintains symmetry in the encoding and decoding halfs. We
Expand All @@ -108,13 +108,11 @@ Consequently, if a type only defines one-half of the symmetry (e.g. it
implements UnmarshalJSON() but not MarshalJSON() ), then that type doesn't
satisfy the check and we will continue walking down the decision tree.
## RPC
RPC Client and Server Codecs are implemented, so the codecs can be used with
the standard net/rpc package.
## Usage
The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent
Expand All @@ -135,85 +133,93 @@ Consequently, the usage model is basically:
Sample usage model:
```go
// create and configure Handle
var (
mh codec.MsgpackHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &mh
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
```
// create and configure Handle
var (
mh codec.MsgpackHandle
)
mh.MapType = reflect.TypeOf(map[string]interface{}(nil))
// configure extensions
// e.g. for msgpack, define functions and enable Time support for tag 1
mh.SetExt(reflect.TypeOf(time.Time{}), 1, myExt)
// create and use decoder/encoder
var (
r io.Reader
w io.Writer
b []byte
h = &mh
)
dec = codec.NewDecoder(r, h)
dec = codec.NewDecoderBytes(b, h)
err = dec.Decode(&v)
enc = codec.NewEncoder(w, h)
enc = codec.NewEncoderBytes(&b, h)
err = enc.Encode(v)
//RPC Server
go func() {
for {
conn, err := listener.Accept()
rpcCodec := codec.GoRpc.ServerCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ServerCodec(conn, h)
rpc.ServeCodec(rpcCodec)
}
}()
//RPC Communication (client side)
conn, err = net.Dial("tcp", "localhost:5555")
rpcCodec := codec.GoRpc.ClientCodec(conn, h)
//OR rpcCodec := codec.MsgpackSpecRpc.ClientCodec(conn, h)
client := rpc.NewClientWithCodec(rpcCodec)
```
## Running Tests
To run tests, use the following:
```
go test
go test
```
To run the full suite of tests, use the following:
```
go test -tags alltests -run Suite
go test -tags alltests -run Suite
```
You can run the tag 'safe' to run tests or build in safe mode. e.g.
```
go test -tags safe -run Json
go test -tags "alltests safe" -run Suite
go test -tags safe -run Json
go test -tags "alltests safe" -run Suite
```
## Running Benchmarks
```
cd codec/bench
./bench.sh -d
./bench.sh -c
./bench.sh -s
go test -bench . -benchmem -benchtime 1s
cd codec/bench
./bench.sh -d
./bench.sh -c
./bench.sh -s
go test -bench . -benchmem -benchtime 1s
```
Please see http://github.com/hashicorp/go-codec-bench .
## Caveats
Struct fields matching the following are ignored during encoding and
Expand Down
13 changes: 7 additions & 6 deletions codec/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
package codec

import (
"encoding/base64"
"encoding/base32"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -132,7 +132,8 @@ var (
errGenAllTypesSamePkg = errors.New("All types must be in the same package")
errGenExpectArrayOrMap = errors.New("unexpected type. Expecting array/map/slice")

genBase64enc = base64.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789__")
// base64 requires 64 unique characters in Go 1.22+, which is not possible for Go identifiers.
genBase32enc = base32.NewEncoding("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdef")
genQNameRegex = regexp.MustCompile(`[A-Za-z_.]+`)
)

Expand Down Expand Up @@ -1829,7 +1830,7 @@ func genMethodNameT(t reflect.Type, tRef reflect.Type) (n string) {
} else {
// best way to get the package name inclusive
// return ptrPfx + strings.Replace(tstr, ".", "_", 1000)
// return ptrPfx + genBase64enc.EncodeToString([]byte(tstr))
// return ptrPfx + genBase32enc.EncodeToString([]byte(tstr))
if t.Name() != "" && genQNameRegex.MatchString(tstr) {
return ptrPfx + strings.Replace(tstr, ".", "_", 1000)
} else {
Expand All @@ -1840,12 +1841,12 @@ func genMethodNameT(t reflect.Type, tRef reflect.Type) (n string) {
}
}

// genCustomNameForType base64encodes the t.String() value in such a way
// genCustomNameForType base32encodes the t.String() value in such a way
// that it can be used within a function name.
func genCustomTypeName(tstr string) string {
len2 := genBase64enc.EncodedLen(len(tstr))
len2 := genBase32enc.EncodedLen(len(tstr))
bufx := make([]byte, len2)
genBase64enc.Encode(bufx, []byte(tstr))
genBase32enc.Encode(bufx, []byte(tstr))
for i := len2 - 1; i >= 0; i-- {
if bufx[i] == '=' {
len2--
Expand Down
8 changes: 8 additions & 0 deletions codec/gen_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
//go:build codecgen.exec

package codec

import "testing"

// TestDontPanic checks that the code compiles with this tag.
func TestDontPanic(t *testing.T) {}

0 comments on commit 1750000

Please sign in to comment.