Skip to content

Commit

Permalink
Record unions and "const"
Browse files Browse the repository at this point in the history
  • Loading branch information
@clemensv
clemensv committed May 2, 2024
1 parent befb6da commit bc44215
Show file tree
Hide file tree
Showing 6 changed files with 272 additions and 46 deletions.
149 changes: 107 additions & 42 deletions lang/csharp/src/apache/main/IO/JsonDecoder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,16 @@
using System;
using System.CodeDom;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using System.Text;
using System.Xml;
using System.Xml.Linq;
using Avro.IO.Parsing;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;

namespace Avro.IO
{
Expand Down Expand Up @@ -244,6 +248,22 @@ public override double ReadDouble()
public override string ReadString()
{
Advance(Symbol.String);
if (Parser.TopSymbol() is Symbol.ConstCheckAction)
{
Symbol.ConstCheckAction top = (Symbol.ConstCheckAction)Parser.PopSymbol();
string expected = (string)top.Value;
if (reader.TokenType != JsonToken.String)
{
throw TypeError("string");
}
string readResult = Convert.ToString(reader.Value);
if (!expected.Equals(readResult))
{
throw new AvroTypeException("Expected constant value: " + expected + " but received: " + readResult);
}
reader.Read();
return readResult;
}
if (Parser.TopSymbol() == Symbol.MapKeyMarker)
{
Parser.Advance(Symbol.MapKeyMarker);
Expand Down Expand Up @@ -734,55 +754,93 @@ public override int ReadUnionIndex()
return n;
}

private bool IsRecordMatch(Symbol symbol, JsonReader objectReader)
private bool IsRecordMatch(Symbol symbol, JTokenReader objectReader)
{
// to determine whether a record matches, we need to read the object and compare it to the schema
// this is done by creating a new JsonDecoder on top of the reader and advancing it through the schema
// if the schema matches, we return true, otherwise false
JsonDecoder innerDecoder = new JsonDecoder(symbol, objectReader, JsonMode.PlainJson);

// the required start condition is that the reader is at the start of the object
// and that the symbol is a Sequence
if ( symbol.SymKind != Symbol.Kind.Sequence || objectReader.CurrentToken.Type != JTokenType.Object)
{
return false;
}
// advance the inner decoder to the start of the record
innerDecoder.Parser.Advance(Symbol.RecordStart);
// read the first token of the object
innerDecoder.reader.Read();
// we're now at the start of the record, so we can start processing the fields
// but we need to do so in the Avro schema field order, so we grab the stack
// of the parser and clone it
var stack = new Stack<Symbol>(innerDecoder.Parser.CloneStack());
try
{

while( objectReader.TokenType != JsonToken.None )
while ( stack.Count > 0 )
{
switch(objectReader.TokenType)
var currentSymbol = stack.Pop();
if (currentSymbol == Symbol.ArrayStart)
{
case JsonToken.PropertyName:
break;
case JsonToken.Integer:
innerDecoder.Advance(Symbol.Int);
break;
case JsonToken.Float:
innerDecoder.Advance(Symbol.Float);
break;
case JsonToken.Boolean:
innerDecoder.Advance(Symbol.Boolean);
break;
case JsonToken.Date:
innerDecoder.Advance(Symbol.JsonDateTime);
break;
case JsonToken.String:
innerDecoder.Advance(Symbol.String);
break;
case JsonToken.Null:
innerDecoder.Advance(Symbol.Null);
break;
case JsonToken.Bytes:
innerDecoder.Advance(Symbol.Bytes);
break;
case JsonToken.StartObject:
innerDecoder.Advance(Symbol.RecordStart);
break;
case JsonToken.EndObject:
break;
case JsonToken.StartArray:
innerDecoder.Advance(Symbol.ArrayStart);
break;
case JsonToken.EndArray:
innerDecoder.Advance(Symbol.ArrayEnd);
break;
default:
break;
innerDecoder.ReadArrayStart();
}
else if (currentSymbol == Symbol.ItemEnd)
{
if ( innerDecoder.ReadArrayNext() == 0 )
{
// pop the repeater
stack.Pop();
}
}
else if ( currentSymbol == Symbol.MapStart)
{
innerDecoder.SkipMap();
innerDecoder.reader.Read();
}
else
{
switch (currentSymbol)
{
case Symbol.FieldAdjustAction fa:
break;
case Symbol.ImplicitAction ia:
break;
case Symbol.Repeater r:
foreach(var s in r.Production)
{
stack.Push(s);
}
break;
default:
innerDecoder.Advance(currentSymbol);
if ( currentSymbol == Symbol.String && stack.Peek() is Symbol.ConstCheckAction)
{
var constCheck = (Symbol.ConstCheckAction)stack.Pop();
if ( innerDecoder.reader.TokenType != JsonToken.String || !constCheck.Check(innerDecoder.reader.Value))
{
return false;
}
}
else
if ((currentSymbol == Symbol.Boolean && innerDecoder.reader.TokenType != JsonToken.Boolean) ||
(currentSymbol == Symbol.Int && innerDecoder.reader.TokenType != JsonToken.Integer) ||
(currentSymbol == Symbol.Long && innerDecoder.reader.TokenType != JsonToken.Integer) ||
(currentSymbol == Symbol.Float && innerDecoder.reader.TokenType != JsonToken.Float) ||
(currentSymbol == Symbol.Double && innerDecoder.reader.TokenType != JsonToken.Float) ||
(currentSymbol == Symbol.String && innerDecoder.reader.TokenType != JsonToken.String) ||
(currentSymbol == Symbol.Bytes && innerDecoder.reader.TokenType != JsonToken.String) ||
(currentSymbol == Symbol.JsonDateTime && innerDecoder.reader.TokenType != JsonToken.Date) ||
(currentSymbol == Symbol.Fixed && innerDecoder.reader.TokenType != JsonToken.String) ||
(currentSymbol == Symbol.Enum && innerDecoder.reader.TokenType != JsonToken.String))
{
return false;
}
innerDecoder.reader.Read();
break;
}
}
objectReader.Read();
}
innerDecoder.Parser.ProcessTrailingImplicitActions();
}
catch (AvroTypeException)
{
Expand Down Expand Up @@ -998,7 +1056,14 @@ public override Symbol DoAction(Symbol input, Symbol top)
.Aggregate((x, y) => x + ", " + y));
}

currentReorderBuffer = reorderBuffers.Pop();
if (reorderBuffers.Count > 0)
{
currentReorderBuffer = reorderBuffers.Pop();
}
else
{
currentReorderBuffer = null;
}
}

// AVRO-2034 advance beyond the end object for the next record.
Expand Down
12 changes: 11 additions & 1 deletion lang/csharp/src/apache/main/IO/Parsing/JsonGrammarGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using System;
using System.Collections.Generic;
using Avro.Util;
using Newtonsoft.Json;

namespace Avro.IO.Parsing
{
Expand Down Expand Up @@ -100,7 +101,16 @@ protected override Symbol Generate(Schema sc, IDictionary<LitS, Symbol> seen)
name = jsonName;
}
production[--i] = new Symbol.FieldAdjustAction(n, name, f.Aliases);
production[--i] = Generate(f.Schema, seen);
string constValue = f.GetProperty("const");
if (constValue != null)
{
var constObj = JsonConvert.DeserializeObject(constValue);
production[--i] = Symbol.NewSeq(new Symbol.ConstCheckAction(constObj), Generate(f.Schema, seen));
}
else
{
production[--i] = Generate(f.Schema, seen);
}
production[--i] = Symbol.FieldEnd;
n++;
}
Expand Down
16 changes: 15 additions & 1 deletion lang/csharp/src/apache/main/IO/Parsing/Parser.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
Expand Down Expand Up @@ -225,5 +225,19 @@ public virtual void Reset()
{
Pos = 1;
}

/// <summary>
/// Clones the stack.
/// </summary>
/// <returns>A copy of the stack</returns>
public Symbol[] CloneStack()
{
var newStack = new Symbol[Pos];
for (int i = 0; i < Pos; i++)
{
newStack[i] = Stack[i];
}
return newStack;
}
}
}
26 changes: 26 additions & 0 deletions lang/csharp/src/apache/main/IO/Parsing/Symbol.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -653,6 +653,32 @@ public IntCheckAction(int size) : base(Kind.ExplicitAction)
}
}

/// <summary>
/// The const check action.
/// </summary>
public class ConstCheckAction : Symbol
{
/// <summary>
/// The value.
/// </summary>
public object Value { get; private set; }

/// <summary>
/// Constructor
/// </summary>
/// <param name="value"></param>
public ConstCheckAction(object value) : base(Kind.ExplicitAction)
{
Value = value;
Production = new Symbol[0];
}

internal bool Check(object value)
{
return Value.Equals(value);
}
}

/// <summary>
/// The writer union action.
/// </summary>
Expand Down
2 changes: 1 addition & 1 deletion lang/csharp/src/apache/main/IO/Parsing/ValidatingGrammarGenerator.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/*
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
Expand Down

0 comments on commit bc44215

Please sign in to comment.