Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

func (*OCFWriter) Append is not appending my data to the avro file #279

Open
HarithaHarman opened this issue Oct 17, 2023 · 1 comment
Open

Comments

@HarithaHarman
Copy link

input.json

package main

import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"

"os"

"github.com/linkedin/goavro/v2"

)

func main() {

jsonFile, err := os.Open("input.json")

if err != nil {
	fmt.Println(err)
}
defer jsonFile.Close()

// read our opened xmlFile as a byte array.
byteValue, _ := ioutil.ReadAll(jsonFile)

// we initialize our Users array
var result map[string]interface{}

// we unmarshal our byteArray which contains our resultset
json.Unmarshal(byteValue, &result)


finalresult := []map[string]interface{}{}
appendresult := []map[string]any{}
for k, v := range result {

	m1 := make(map[string]interface{})
	m2 := make(map[string]any)
	m1["name"] = k
	switch v.(type) {
	case string:
		m1["type"] = "string"
		m2[k] = v
		appendresult = append(appendresult, m2)
	case float64:
		m1["type"] = "long"
		m2[k] = v
		appendresult = append(appendresult, m2)

	default:
		m1["type"] = "null"
		m2[k] = result[k]
		appendresult = append(appendresult, m2)
	}

	finalresult = append(finalresult, m1)
}


jsonString, _ := json.Marshal(finalresult)



var fields []Field
json.Unmarshal(jsonString, &fields)

type recordschema struct {
	Name   string  `json:"name"`
	Type   string  `json:"type"`
	Fields []Field `json:"fields"`
}
group := recordschema{
	Name:   "record",
	Type:   "record",
	Fields: fields,
}
b, err := json.Marshal(group)

codec, err := goavro.NewCodec(string(b))
if err != nil {
	fmt.Println(err)
}

var ocfFileContents bytes.Buffer
writer, err := goavro.NewOCFWriter(goavro.OCFConfig{
	W:      &ocfFileContents,
	Schema: codec.Schema(),
})
if err != nil {
	fmt.Println(err)
}
err = writer.Append(appendresult)

fmt.Println(ocfFileContents.String())
file, err := os.Create("output16.avro")
if err != nil {
	panic(err)
}
defer file.Close()

// Write the Avro binary data to the file
_, err = file.Write(ocfFileContents.Bytes())
if err != nil {
	panic(err)
}

}

type Field struct {
Name string json:"name"
Type string json:"type"
}
avrofile

@HarithaHarman
Copy link
Author

Hi Team, I am trying to convert json file to avro file. after convertion the data is not appending properly as marked in the above image. could you please help me

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant