Skip to content

Commit

Permalink
switch csv and tsv method 'sv' from ReadAll() to stream each record w…
Browse files Browse the repository at this point in the history 
…ith Read(), to improve memory usage. related to issue #354
  • Loading branch information
@viprerk
viprerk committed Dec 16, 2022
1 parent d9466a5 commit fe45260
Show file tree
Hide file tree
Showing 4 changed files with 282 additions and 2 deletions.
16 changes: 14 additions & 2 deletions internal/magic/text_csv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package magic
import (
"bytes"
"encoding/csv"
"errors"
"io"
)

Expand All @@ -23,8 +24,19 @@ func sv(in []byte, comma rune, limit uint32) bool {
r.LazyQuotes = true
r.Comment = '#'

lines, err := r.ReadAll()
return err == nil && r.FieldsPerRecord > 1 && len(lines) > 1
lines := 0
for {
_, err := r.Read()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return false
}
lines++
}

return r.FieldsPerRecord > 1 && lines > 1
}

// dropLastLine drops the last incomplete line from b.
Expand Down
2 changes: 2 additions & 0 deletions mimetype_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ var files = map[string]string{
"class.class": "application/x-java-applet",
"crx.crx": "application/x-chrome-extension",
"csv.csv": "text/csv",
"csv_long.csv": "text/csv",
"cpio.cpio": "application/x-cpio",
"dae.dae": "model/vnd.collada+xml",
"dbf.dbf": "application/x-dbf",
Expand Down Expand Up @@ -200,6 +201,7 @@ var files = map[string]string{
"tiff.tiff": "image/tiff",
"torrent.torrent": "application/x-bittorrent",
"tsv.tsv": "text/tab-separated-values",
"tsv_long.tsv": "text/tab-separated-values",
"ttc.ttc": "font/collection",
"ttf.ttf": "font/ttf",
"tzfile": "application/tzif",
Expand Down
47 changes: 47 additions & 0 deletions testdata/csv_long.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# some comments
Start Date ,Start Time,End Date,End Time,Event Title ,All Day Event,No End Time,Event Description,Contact ,Contact Email,Contact Phone,Location,Category,Mandatory,Registration,Maximum,Last Date To Register
# hello
9/5/2011,3:00:00 PM,9/5/2011,,Social Studies Dept. Meeting,N,Y,Department meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
# see ya
9/5/2011,6:00:00 PM,9/5/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/6/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/7/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/8/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/9/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/10/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/11/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/12/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/13/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/14/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/15/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/16/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/17/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/18/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/19/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/20/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/21/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/22/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/23/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/24/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,9/25/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/5/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/6/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/7/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/8/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/9/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/10/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/11/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/12/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/13/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/14/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/15/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/16/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/17/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/18/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/19/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/20/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/21/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/22/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/23/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/24/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
9/5/2011,6:00:00 PM,10/25/2011,8:00:00 PM,Curriculum Meeting,N,N,Curriculum Meeting,Chris Gallagher,cgallagher@schoolwires.com,814-555-5179,High School,2,N,N,25,9/2/2011
219 changes: 219 additions & 0 deletions testdata/tsv_long.tsv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
help direct great nearly chair realize
probably examine steady serve would reason
sang about pair dirty heavy reader
production increase chance contrast my nine
best dropped arrow victory tribe knowledge
lie should sort applied talk smallest
world rear product seeing substance tank
ahead sale unusual closely importance fed
away close ran slip bridge protection
fall parent flew syllable offer motor
by valuable cause herself plain flame
strip nails carbon island college canal
organization paragraph heat directly throughout mouth
cook purpose speed everywhere leaving tobacco
flower friendly rising clear rhyme partly
least gather sand milk hearing mother
broken design read shinning bottom wish
similar parent spring care east else
rubber slowly board weight rubber damage
sick mission movement pay route more
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
involved author size cotton half apple
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow
upper rule smell post neighbor orbit
have planned quite sitting call merely
medicine ring theory discovery calm this
accident great prove garden buried dropped
jar needed sound driving common medicine
east search create journey mostly gun
aloud race plates black giant throat
fair dinner chicken metal help mark
goes also language position hole save
basic tree palace am hard theory
mad suggest chosen due due most
went double rope recent evidence chair
able twelve hospital road stock stretch
sweet name dirt settlers sheet needle
tell pencil on command onlinetools must
rise milk read hour death valley
amount determine massage pack image disappear
suppose tell club national chair somehow

0 comments on commit fe45260

Please sign in to comment.