Python enable markdown acceptance tests

python/Makefile

@@ -8,8 +8,8 @@ SOURCE_FILES = $(shell find . -name "*.py" | grep -v $(GHERKIN_PARSER))
 GHERKIN = bin/gherkin
 GHERKIN_GENERATE_TOKENS = bin/gherkin-generate-tokens
 
-GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature")
-BAD_FEATURE_FILES  = $(shell find ../testdata/bad -name "*.feature")
+GOOD_FEATURE_FILES = $(shell find ../testdata/good -name "*.feature" -o -name "*.feature.md")
+BAD_FEATURE_FILES  = $(shell find ../testdata/bad -name "*.feature" -o -name "*.feature.md")
 
 TOKENS       = $(patsubst ../testdata/%,acceptance/testdata/%.tokens,$(GOOD_FEATURE_FILES))
 ASTS         = $(patsubst ../testdata/%,acceptance/testdata/%.ast.ndjson,$(GOOD_FEATURE_FILES))

python/bin/gherkin_generate_tokens.py

@@ -7,12 +7,18 @@
 from gherkin.token_scanner import TokenScanner
 from gherkin.token_formatter_builder import TokenFormatterBuilder
 from gherkin.parser import Parser
+from gherkin.token_matcher_markdown import GherkinInMarkdownTokenMatcher
 
 files = sys.argv[1:]
 if sys.version_info < (3, 0) and os.name != 'nt':  # for Python2 unless on Windows native
     UTF8Writer = codecs.getwriter('utf8')
     sys.stdout = UTF8Writer(sys.stdout)
+
 parser = Parser(TokenFormatterBuilder())
 for file in files:
     scanner = TokenScanner(file)
-    print(parser.parse(scanner))
+
+    if(file.endswith('.md')):
+        print(parser.parse(scanner, GherkinInMarkdownTokenMatcher()) )
+    else:
+        print(parser.parse(scanner))

python/gherkin/parser.py

@@ -250,6 +250,11 @@ def match_token_at_0(self, token, context):
         if self.match_Empty(context, token):
                 self.build(context, token)
                 return 0
+        # FIXME - move into gherkin-python.razor
+        if self.match_ScenarioLine(context,token):
+                self.build(context, token)
+                return 12
+
 
         state_comment = "State: 0 - Start"
         token.detach

python/gherkin/token_matcher_markdown.py

@@ -82,8 +82,9 @@ def match_Comment(self, token):
         if(token.line.startswith('|')):
             table_cells = token.line.table_cells
             if(self._is_gfm_table_separator(table_cells)):
+                self._set_token_matched(token,"Comment")
                 return True
-        return self._set_token_matched(token,None,False)
+        return self._set_token_matched(token,None)
 
     def match_Empty(self, token):
 

python/test/gherkin_in_markdown_token_matcher_test.py

@@ -6,7 +6,16 @@
 from gherkin.gherkin_line import GherkinLine
 location = { 'line': 1, 'column': 1 }
 
-def test_it_matches_FeatureLine():
+def test_it_matches_FeatureLineH1():
+    tm = GherkinInMarkdownTokenMatcher('en')
+    line = GherkinLine('''# Feature: hello''',location['line'])
+    token = Token(gherkin_line=line, location=location)
+    assert tm.match_FeatureLine(token)
+    assert token.matched_type == 'FeatureLine'
+    assert token.matched_keyword == 'Feature'
+    assert token.matched_text == 'hello'
+
+def test_it_matches_FeatureLineH2():
     tm = GherkinInMarkdownTokenMatcher('en')
     line = GherkinLine('''## Feature: hello''',location['line'])
     token = Token(gherkin_line=line, location=location)
@@ -151,11 +160,13 @@ def test_it_matches_table_separator_row_as_comment():
     l1 = GherkinLine('  | h1 | h2 |',location['line'])
     t1 = Token(l1,location)
     assert tm.match_TableRow(t1)
+    assert t1.location['column'] == 3
 
     l2 = GherkinLine('  | --- | --- |',location['line'])
     t2 = Token(l2,location)
     assert not tm.match_TableRow(t2)
     assert tm.match_Comment(t2)
+    assert t2.location['column'] == 3
 
 def test_it_matches_indented_tags():
     tm = GherkinInMarkdownTokenMatcher('en')
@@ -229,4 +240,13 @@ def test_it_matches_ExamplesLine():
     assert tm.match_ExamplesLine(token)
     assert token.matched_type == 'ExamplesLine'
     assert token.matched_keyword == 'Examples'
-    assert token.matched_text == ''
+    assert token.matched_text == ''
+
+def test_it_matches_Empty():
+    tm = GherkinInMarkdownTokenMatcher('en')
+    line = GherkinLine('''''',location['line'])
+    token = Token(gherkin_line=line, location=location)
+    assert tm.match_Empty(token)
+    assert token.matched_type == 'Empty'
+    assert token.matched_keyword == None
+    assert token.matched_text == None

testdata/good/datatables.feature.md.tokens

@@ -0,0 +1,9 @@
+(1:4)FeatureLine:()Feature/DataTables/
+(2:1)Empty://
+(3:5)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:3)StepLine:()Given /a simple data table/
+(6:3)TableRow:()|//5:foo,11:bar
+(7:3)Comment://
+(8:3)TableRow:()|//5:boz,11:boo
+EOF

testdata/good/docstrings.feature.md.tokens

@@ -0,0 +1,9 @@
+(1:4)FeatureLine:()Feature/DocString variations/
+(2:1)Empty://
+(3:5)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:3)StepLine:()And /a DocString with an implicitly escaped separator inside/
+(6:1)DocStringSeparator:()````//
+(7:1)Other:/```/
+(8:1)DocStringSeparator:()````//
+EOF

testdata/good/minimal.feature.md

@@ -2,4 +2,4 @@
 
 ## Scenario: minimalistic
 
-  *  Given the minimalism
+* Given the minimalism

testdata/good/minimal.feature.md.tokens

@@ -0,0 +1,6 @@
+(1:3)FeatureLine:()Feature/Minimal/
+(2:1)Empty://
+(3:4)ScenarioLine:()Scenario/minimalistic/
+(4:1)Empty://
+(5:3)StepLine:()Given /the minimalism/
+EOF

testdata/good/misc.feature.md.tokens

@@ -0,0 +1,26 @@
+(1:1)Empty://
+(2:1)Empty://
+(3:1)Empty://
+(4:1)Empty://
+(5:1)Empty://
+(6:1)Empty://
+(7:1)Empty://
+(8:1)Empty://
+(9:1)Empty://
+(10:1)Empty://
+(11:1)Empty://
+(12:3)ScenarioLine:()Scenario/Something about math/
+(13:3)StepLine:()Given /step one/
+(14:3)StepLine:()When /step two/
+(15:3)StepLine:()Then /step three/
+(16:1)Empty://
+(17:3)ScenarioLine:()Scenario/Something about gravity/
+(18:4)StepLine:()Given /step one/
+(19:4)StepLine:()When /step two/
+(20:4)StepLine:()Then /step three/
+(21:1)Empty://
+(22:1)Empty://
+(23:1)Empty://
+(24:1)Empty://
+(25:1)Empty://
+EOF

testdata/good/tags.feature.md.tokens

@@ -0,0 +1,43 @@
+(1:1)TagLine://2:@feature_tag1,18:@feature_tag2
+(2:3)TagLine://4:@feature_tag3
+(3:3)FeatureLine:()Feature/Minimal Scenario Outline/
+(4:1)Empty://
+(5:1)TagLine://2:@scenario_tag1,19:@scenario_tag2
+(6:3)TagLine://4:@scenario_tag3
+(7:4)ScenarioLine:()Scenario/minimalistic/
+(8:3)StepLine:()Given /the minimalism/
+(9:1)Empty://
+(10:1)TagLine://2:@so_tag1,14:@so_tag2
+(11:3)TagLine://4:@so_tag3
+(12:4)ScenarioLine:()Scenario Outline/minimalistic outline/
+(13:3)StepLine:()Given /the <what>/
+(14:1)Empty://
+(15:1)TagLine://2:@ex_tag1,13:@ex_tag2
+(16:3)TagLine://4:@ex_tag3
+(17:5)ExamplesLine:()Examples//
+(18:3)TableRow:()|//5:what
+(19:3)Comment://
+(20:3)TableRow:()|//5:minimalism
+(21:1)Empty://
+(22:1)TagLine://2:@ex_tag4,13:@ex_tag5
+(23:3)TagLine://4:@ex_tag6
+(24:5)ExamplesLine:()Examples//
+(25:3)TableRow:()|//5:what
+(26:3)Comment://
+(27:3)TableRow:()|//5:more minimalism
+(28:1)Empty://
+(29:1)TagLine://2:@comment_tag1
+(30:4)ScenarioLine:()Scenario/comments/
+(31:1)Empty://
+(32:1)Empty://
+(33:1)TagLine://2:@comment_tag#2
+(34:4)ScenarioLine:()Scenario/hash in tags/
+(35:1)Empty://
+(36:1)Empty://
+(37:1)TagLine://2:@rule_tag
+(38:4)RuleLine:()Rule//
+(39:1)Empty://
+(40:1)TagLine://2:@joined_tag3,16:@joined_tag4
+(41:5)ScenarioLine:()Scenario/joined tags/
+(42:1)Empty://
+EOF