Skip to content

Commit

Permalink
Sheets -> CSV transform fixes (#3)
Browse files Browse the repository at this point in the history
  • Loading branch information
@joecorall
joecorall authored Jul 25, 2024
1 parent 2fc09a2 commit 3172321
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 43 deletions.
5 changes: 3 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ sequenceDiagram
## Getting started

```
go install github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen@latest
go generate ./api
go run main.go \
--source /path/to/google/sheet.csv \
--target workbench.csv
```


Expand Down
19 changes: 18 additions & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,36 @@ go 1.22.2

require (
github.com/golang-jwt/jwt/v5 v5.2.1
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240709193244-50f8e60d633d
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240715123742-714adfa865a9
github.com/lestrrat-go/jwx v1.2.29
github.com/sfomuseum/go-edtf v1.1.1
)

require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
github.com/getkin/kin-openapi v0.124.0 // indirect
github.com/go-openapi/jsonpointer v0.20.2 // indirect
github.com/go-openapi/swag v0.22.8 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/invopop/yaml v0.2.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
github.com/lestrrat-go/blackmagic v1.0.2 // indirect
github.com/lestrrat-go/httpcc v1.0.1 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
github.com/oapi-codegen/oapi-codegen/v2 v2.3.0 // indirect
github.com/perimeterx/marshmallow v1.1.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/spf13/cobra v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.18.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/tools v0.22.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
37 changes: 37 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,30 @@
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0=
github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M=
github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM=
github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw=
github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY=
github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240709193244-50f8e60d633d h1:bryeojGZvWZazzCdVsZB0Pi3LijX/Aq82KgDnjPkt0s=
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240709193244-50f8e60d633d/go.mod h1:JDCARba/UJW608jcs6XyVuCsfp3LoDVDC++bnGAB47A=
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240715123742-714adfa865a9 h1:d6uCv/YmDxtDUyzVfTlVUBciSm5vPR9DrZKn+dKqPMo=
github.com/lehigh-university-libraries/go-islandora v0.0.0-20240715123742-714adfa865a9/go.mod h1:JDCARba/UJW608jcs6XyVuCsfp3LoDVDC++bnGAB47A=
github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k=
Expand All @@ -23,12 +38,25 @@ github.com/lestrrat-go/jwx v1.2.29/go.mod h1:hU8k2l6WF0ncx20uQdOmik/Gjg6E3/wIRtX
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw=
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
github.com/oapi-codegen/oapi-codegen/v2 v2.3.0 h1:rICjNsHbPP1LttefanBPnwsSwl09SqhCO7Ee623qR84=
github.com/oapi-codegen/oapi-codegen/v2 v2.3.0/go.mod h1:4k+cJeSq5ntkwlcpQSxLxICCxQzCL772o30PxdibRt4=
github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s=
github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sfomuseum/go-edtf v1.1.1 h1:R5gElndHGDaK/rGSh2X+ulaLtlcHCdQA1cTzB8e9wv8=
github.com/sfomuseum/go-edtf v1.1.1/go.mod h1:1rP0EJZ/84j3HO80vGcnG2T9MFBDAFyTNtjrr8cv3T4=
github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
Expand All @@ -47,6 +75,8 @@ golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
Expand Down Expand Up @@ -77,12 +107,19 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
157 changes: 117 additions & 40 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import (
"net/http"
"os"
"reflect"
"regexp"
"strconv"
"strings"

"github.com/lehigh-university-libraries/fabricator/internal/handlers"
Expand All @@ -21,20 +23,21 @@ func getJSONFieldName(tag string) string {
return tag
}

func readCSVWithJSONTags(filePath string) ([]map[string][]string, error) {
func readCSVWithJSONTags(filePath string) (map[string]bool, []map[string][]string, error) {
file, err := os.Open(filePath)
if err != nil {
return nil, err
return nil, nil, err
}
defer file.Close()

re := regexp.MustCompile(`^\d{3}$`)
reader := csv.NewReader(file)
headers, err := reader.Read()
if err != nil {
return nil, err
return nil, nil, err
}

var rows []map[string][]string
newHeaders := map[string]bool{}
newCsv := &workbench.SheetsCsv{}
for {
record, err := reader.Read()
Expand All @@ -54,11 +57,77 @@ func readCSVWithJSONTags(filePath string) ([]map[string][]string, error) {
value := v.FieldByName(field.Name)
if value.IsValid() && value.CanSet() {
str := record[i]
if str == "" {
continue
}

column := getJSONFieldName(field.Tag.Get("csv"))
switch header {
if column == "" {
return nil, nil, fmt.Errorf("unknown column: %s", jsonTag)
}
switch column {
case "field_add_coverpage", "published":
switch str {
case "Yes":
str = "1"
case "No":
str = "0"
default:
return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str)
}
case "id", "parent_id":
if !re.MatchString(str) {
return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str)
}
case "field_weight":
_, err := strconv.Atoi(str)
if err != nil {
return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str)
}
case "field_subject_hierarchical_geo":
str = `{"country":"United States","state":"Pennsylvania","county":"Lehigh","city":"Coplay"}`
case "field_rights":
switch str {
case "IN COPYRIGHT":
str = "http://rightsstatements.org/vocab/InC/1.0/"
case "IN COPYRIGHT - EU ORPHAN WORK":
str = "http://rightsstatements.org/vocab/InC-OW-EU/1.0/"
case "IN COPYRIGHT - EDUCATIONAL USE PERMITTED":
str = "http://rightsstatements.org/vocab/InC-EDU/1.0/"
case "IN COPYRIGHT - NON-COMMERCIAL USE PERMITTED":
str = "http://rightsstatements.org/vocab/InC-NC/1.0/"
case "IN COPYRIGHT - RIGHTS-HOLDER(S) UNLOCATABLE OR UNIDENTIFIABLE":
str = "http://rightsstatements.org/vocab/InC-RUU/1.0/"
case "NO COPYRIGHT - CONTRACTUAL RESTRICTIONS":
str = "http://rightsstatements.org/vocab/NoC-CR/1.0/"
case "NO COPYRIGHT - NON-COMMERCIAL USE ONLY":
str = "http://rightsstatements.org/vocab/NoC-NC/1.0/"
case "NO COPYRIGHT - OTHER KNOWN LEGAL RESTRICTIONS":
str = "http://rightsstatements.org/vocab/NoC-OKLR/1.0/"
case "NO COPYRIGHT - UNITED STATES":
str = "http://rightsstatements.org/vocab/NoC-US/1.0/"
case "COPYRIGHT NOT EVALUATED":
str = "http://rightsstatements.org/vocab/CNE/1.0/"
case "COPYRIGHT UNDETERMINED":
str = "http://rightsstatements.org/vocab/UND/1.0/"
case "NO KNOWN COPYRIGHT":
str = "http://rightsstatements.org/vocab/NKC/1.0/"
default:
return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str)
}
case "field_linked_agent.vid":
if str == "Corporate Body" {
str = "corporate_body"
} else if str == "Person" {
str = "person"
} else if str == "Family" {
str = "family"
} else {
return nil, nil, fmt.Errorf("unknown %s: %s", jsonTag, str)
}
case "field_linked_agent.rel_type":
components := strings.Split(str, "|")
str = components[1]
str = components[0]
case "field_extent.attr0=page",
"field_extent.attr0=dimensions",
"field_extent.attr0=bytes",
Expand All @@ -79,32 +148,29 @@ func readCSVWithJSONTags(filePath string) ([]map[string][]string, error) {
"field_identifier.attr0=uri",
"field_identifier.attr0=call-number",
"field_identifier.attr0=report-number":
components := strings.Split(header, ".attr0=")
components := strings.Split(column, ".attr0=")
column = components[0]
str = fmt.Sprintf(`{"value":"%s","attr0":"%s"}`, str, components[1])
case "field_geographic_subject.vid=geographic_naf",
"field_geographic_subject.vid=geographic_local":
components := strings.Split(header, ".vid=")
components := strings.Split(column, ".vid=")
column = components[0]
str = fmt.Sprintf("%s:%s", components[1], str)
/*
case "field_related_item.title":
case "field_related_item.identifier_type=issn":
case "field_linked_agent.vid",
"field_linked_agent.rel_type":
*/
case "File Path":
// TODO case "field_related_item.title":
// TODO case "field_related_item.identifier_type=issn":
case "file":
str = strings.ReplaceAll(str, `\`, `/`)
str = strings.TrimLeft(str, "/")
if len(str) > 3 && str[0:3] != "mnt" {
str = fmt.Sprintf("/mnt/islandora_staging/%s", str)
}
}

newHeaders[column] = true
// replace the locally defined google sheets cell delimiter
// with workbench's pipe delimiter
str = strings.ReplaceAll(str, " ; ", "|")
if str != "" {
row[column] = append(row[column], str)
}
row[column] = append(row[column], str)
}
}
}
Expand All @@ -113,7 +179,7 @@ func readCSVWithJSONTags(filePath string) ([]map[string][]string, error) {
rows = append(rows, row)
}

return rows, nil
return newHeaders, rows, nil
}

func main() {
Expand Down Expand Up @@ -142,7 +208,7 @@ func main() {
flag.Usage()
return
}
rows, err := readCSVWithJSONTags(*source)
headers, rows, err := readCSVWithJSONTags(*source)
if err != nil {
slog.Error("Failed to read CSV", "err", err)
os.Exit(1)
Expand All @@ -158,40 +224,42 @@ func main() {
writer := csv.NewWriter(file)
defer writer.Flush()

// get all possible headers in the CSV
headers := []string{}
for header := range rows[0] {
headers = append(headers, header)
}

// check any columns that have no values
includeColumns := map[string]bool{}
for _, row := range rows {
for _, header := range headers {
if !includeColumns[header] && len(row[header]) > 0 {
includeColumns[header] = true
for k := range rows {
for header := range headers {
if header == "field_linked_agent.name" {
delete(headers, header)
name := rows[k][header]
if len(name) == 0 {
continue
}
header = "field_linked_agent"
headers[header] = true
vid := rows[k]["field_linked_agent.vid"]
rel := rows[k]["field_linked_agent.rel_type"]
rows[k][header] = []string{
fmt.Sprintf("%s:%s:%s", rel[0], vid[0], name[0]),
}
} else if header == "field_linked_agent.rel_type" || header == "field_linked_agent.vid" {
delete(headers, header)
}
}
}

// remove columns with no values from the header
headers = []string{}
for header, include := range includeColumns {
if include {
headers = append(headers, header)
}
firstRow := make([]string, 0, len(headers))
for header := range headers {
firstRow = append(firstRow, header)
}

// finally, write the header to the CSV
if err := writer.Write(headers); err != nil {
if err := writer.Write(firstRow); err != nil {
slog.Error("Failed to write record to CSV", "err", err)
os.Exit(1)
}

// write the rows to the CSV
for _, row := range rows {
record := []string{}
for _, header := range headers {
for _, header := range firstRow {
record = append(record, strings.Join(row[header], "|"))
}
if err := writer.Write(record); err != nil {
Expand All @@ -202,3 +270,12 @@ func main() {

slog.Info("CSV file has been written successfully")
}

func StrInSlice(s string, sl []string) bool {
for _, a := range sl {
if a == s {
return true
}
}
return false
}

0 comments on commit 3172321

Please sign in to comment.