"""Tests the interface Modules/DB"""

from uuid import uuid4 as gen_uuid
import pytest

from bemserver.database import ServiceDB, ModelDB, OutputDB
from bemserver.database.exceptions import ItemNotFoundError
from bemserver.models import (
    Service, Model, OutputEvent, OutputTimeSeries, ValuesDescription,
    Parameter)

from tests import TestCoreDatabaseOntology

@pytest.mark.usefixtures('init_onto_mgr_fact')
class TestServiceDB(TestCoreDatabaseOntology):
    """Tests on the interface to handle services in the ontology."""

    def test_db_service_get_empty(self):

        service_db = ServiceDB()

        # get all items
        result = service_db.get_all()
        assert list(result) == []

        # try to get an inexistant item
        with pytest.raises(ItemNotFoundError):
            service_db.get_by_id('not_existing')

    def test_db_service_create(self, init_sites):

        service_db = ServiceDB()

        # check that database is empty
        result = service_db.get_all()
        assert list(result) == []

        # create an item

        service = Service('Service#0', url='http://example.com',
                          description='New sample service',
                          has_frontend=True, site_ids=init_sites)
        new_service_id = service_db.create(service)
        assert new_service_id is not None
        assert new_service_id == service.id

        # check that database is not empty now
        result = service_db.get_all()
        services = list(result)
        assert len(services) == 1
        assert services[0].id == service.id
        assert services[0].name == service.name
        assert services[0].description == service.description
        assert services[0].url == service.url
        assert services[0].has_frontend
        assert set(services[0].site_ids) == set(init_sites)

    def test_db_service_filter(self, init_sites):
        service_db = ServiceDB()

        # check that database is empty
        result = service_db.get_all(name='Service#0')
        assert list(result) == []
        result = service_db.get_all(
            url="http://example.com")
        assert list(result) == []

        # create an item
        service_db.create(
            Service('Service#0', url='http://example.com',
                    description='New sample FDD service',
                    site_ids=[str(init_sites[0])]))
        service_db.create(
            Service('Service#1', url='http://hit2gap.eu/services/forecasting',
                    description='New sample forecasting service',
                    has_frontend=True))

        services = list(service_db.get_all())
        assert len(services) == 2
        result = service_db.get_all(name="Service#1")
        assert len(list(result)) == 1

        result = service_db.get_all(
            url="http://hit2gap.eu/services/forecasting")
        assert len(list(result)) == 1

        result = service_db.get_all(site=str(init_sites[0]))
        assert len(list(result)) == 1

        result = service_db.get_all(has_frontend=True)
        assert len(list(result)) == 1

    def test_db_service_update(self, init_services):

        service_ids = init_services
        service_db = ServiceDB()

        # get all items
        result = service_db.get_all()
        services = list(result)
        assert len(services) == 2
        for cur_service in services:
            assert cur_service.id in service_ids

        # get an item by its ID!
        # for a service, the ID to be used is the URL
        service = service_db.get_by_id(services[0].id)

        # update item data
        new_description = 'updated by patator'
        new_name = 'New Name'
        new_url = 'http://example.com'

        service.description = new_description
        service.name = new_name
        service.url = new_url
        service_db.update(service.id, service)

        # check that item has really been updated in database
        updated_service = service_db.get_by_id(service.id)
        assert updated_service.id == service.id
        assert updated_service.name == new_name
        assert updated_service.description == new_description
        assert updated_service.url == new_url
        assert updated_service.kind == service.kind == 'Service'
        assert updated_service.model_ids == service.model_ids == []

        # delete an item by its ID
        service_db.remove(service.id)

        # get an item by its ID
        with pytest.raises(ItemNotFoundError):
            # it has been removed...
            service_db.get_by_id(service.id)

@pytest.mark.usefixtures('init_onto_mgr_fact')
class TestModelDB(TestCoreDatabaseOntology):
    """Tests on the interface to handle services in the ontology."""

    def test_db_model_get_empty(self):

        model_db = ModelDB()

        # get all items
        result = model_db.get_all()
        assert list(result) == []

        # try to get an inexistant item
        with pytest.raises(ItemNotFoundError):
            model_db.get_by_id('not_existing')

    def test_db_model_create(self, init_services):

        model_db = ModelDB()

        # check that database is empty
        result = model_db.get_all()
        assert list(result) == []

        # create an item
        model = Model('ARIMA#0', init_services[0],
                      description='A sample model',
                      parameters=[Parameter('ar', 5)])
        new_model_id = model_db.create(model)
        assert new_model

        assert new_model_id == model    .id

        # check that database is not empty now
        result = model_db.get_all()
        models = list(result)
        assert len(models) == 1
        assert models[0].id == model.id
        assert models[0].name == model.name
        assert models[0].service_id == model.service_id
        assert models[0].description == model.description
        assert models[0].get_set_of_parameters() == (
            model.get_set_of_parameters())

        # test relation between service and model
        service = ServiceDB().get_by_id(model.service_id)
        assert service.model_ids == [model.id]

    def test_db_model_filter(self, init_services):
        model_db = ModelDB()
        services = init_services

        # check that database is empty
        result = model_db.get_all(name='Model #0')
        assert list(result) == []
        result = model_db.get_all(service_id=str(services[0]))
        assert list(result) == []

        # create an item
        model_db.create(
            Model('Model #0', services[0], description='New sample FDD model',
                  parameters=[Parameter('param1', 1)]))
        model_db.create(
            Model('Model #1', services[0],
                  description='RENew sample FDD model',
                  parameters=[Parameter('param1', 1), Parameter('param2', 2)]))

        models = list(model_db.get_all())
        assert len(models) == 2
        result = model_db.get_all(name="Model #1")
        assert len(list(result)) == 1

        result = model_db.get_all(service_id=services[0])
        assert len(list(result)) == 2

        result = model_db.get_all(service_id=services[1])
        assert len(list(result)) == 0

    def test_db_model_update(self, init_models):
        model_ids, _ = init_models
        model_db = ModelDB()

        # get all items
        result = model_db.get_all()
        models = list(result)
        assert len(models) == 2
        for cur_model in models:
            assert cur_model.id in model_ids

        # get an item by its ID!
        # for a model, the ID to be used is the URL
        model = model_db.get_by_id(models[0].id)

        # update item data
        new_description = 'updated by patator'
        new_name = 'New Name'
        new_parameters = [Parameter('NewParam', 0)]
        model.description = new_description
        model.name = new_name
        model.parameters = new_parameters
        model_db.update(model.id, model)

        # check that item has really been updated in database
        updated_model = model_db.get_by_id(model.id)
        assert updated_model.id == model.id
        assert updated_model.name == new_name
        assert updated_model.description == new_description
        assert updated_model.service_id == model.service_id
        assert set(updated_model.event_output_ids) ==\
            set(model.event_output_ids)
        assert set(updated_model.timeseries_output_ids) ==\
            set(model.timeseries_output_ids)

        assert updated_model.get_set_of_parameters() ==\
            model.get_set_of_parameters() == set([('NewParam', 0)])

        # delete an item by its ID
        model_db.remove(model.id)

        # get an item by its ID
        with pytest.raises(ItemNotFoundError):
            # it has been removed...
            model_db.get_by_id(model.id)

@pytest.mark.usefixtures('init_onto_mgr_fact')
class TestOutputDB(TestCoreDatabaseOntology):
    """Tests on the interface to handle services in the ontology."""

    def test_db_output_get_empty(self):
        output_db = OutputDB()
        # get all items
        result = output_db.get_all()
        assert list(result) == []
        # try to get an inexistant item
        with pytest.raises(ItemNotFoundError):
            output_db.get_by_id('not_existing')

    def test_db_output_create(self, init_models, init_buildings):
        building_ids, _ = init_buildings
        output_db = OutputDB()
        # check that database is empty
        result = output_db.get_all()
        assert list(result) == []

        # get service and model id
        models = list(ModelDB().get_all())
        model_id, service_id = models[0].id, models[0].service_id

        # create an item
        output = OutputEvent(service_id, model_id, id=str(gen_uuid()))
        new_output_id_evt = output_db.create(output)
        assert new_output_id_evt is not None

        # check that database is not empty now
        result = output_db.get_all()
        outputs = list(result)
        assert len(outputs) == 1
        assert outputs[0].module_id == service_id
        assert outputs[0].model_id == model_id

        # test with time series
        output = OutputTimeSeries(
            service_id, model_id, building_ids[0],
            ValuesDescription('Temperature', 'DegreeCelsius', 20),
            id=str(gen_uuid()))
        new_output_id_ts = output_db.create(output)
        assert new_output_id_ts is not None

        result = output_db.get_all()
        outputs = list(result)
        assert len(outputs) == 2
        output_event = (
            outputs[0]
            if isinstance(outputs[0], OutputTimeSeries) else outputs[1])
        assert output_event.module_id == service_id
        assert output_event.model_id == model_id
        assert output_event.localization == output.localization
        assert output_event.values_desc.unit == output.values_desc.unit
        assert output_event.values_desc.kind == output.values_desc.kind
        assert output_event.values_desc.sampling == output.values_desc.sampling

        # test relation between service and output
        model = ModelDB().get_by_id(model_id)
        assert set(model.event_output_ids) == {new_output_id_evt}
        assert set(model.timeseries_output_ids) == {new_output_id_ts}

    def test_db_output_filter(self, init_models, init_buildings):
        building_ids, _ = init_buildings
        output_db = OutputDB()

        # get service and model id
        models = list(ModelDB().get_all())
        model_id, service_id = models[0].id, models[0].service_id

        # check that database is empty
        result = output_db.get_all(module_id=service_id)
        assert list(result) == []
        result = output_db.get_all(kind='Temperature')
        assert list(result) == []

        # create an item
        output_db.create(OutputEvent(service_id, model_id))
        output_db.create(
            OutputTimeSeries(
                service_id, model_id, building_ids[0],
                ValuesDescription('Temperature', 'DegreeCelsius', 20)))

        outputs = list(output_db.get_all())
        assert len(outputs) == 2
        result = output_db.get_all(module_id=service_id)
        assert len(list(result)) == 2

        result = output_db.get_all(localization=building_ids[0])
        assert len(list(result)) == 1

        result = output_db.get_all(value_type='Temperature')
        assert len(list(result)) == 1

    def test_db_output_update(self, init_models, init_buildings):
        building_ids, _ = init_buildings

        models = list(ModelDB().get_all())
        model_id, service_id = models[0].id, models[0].service_id

        output_db = OutputDB()
        output = OutputTimeSeries(
            service_id, model_id, building_ids[0],
            ValuesDescription('Temperature', 'DegreeCelsius', 20),
            id=str(gen_uuid()))
        new_output_id_ts = output_db.create(output)

        # get all items
        result = output_db.get_all()
        outputs = list(result)
        assert len(outputs) == 1
        assert [item.id for item in outputs] == [new_output_id_ts]

        # get an item by its ID!
        # for a output, the ID to be used is the URL
        output = output_db.get_by_id(outputs[0].id)

        # update item data
        new_sampling = 400
        new_model_id = models[1].id
        new_kind = 'Energy'
        new_unit = 'Joule'
        output.sampling = new_sampling
        output.model_id = new_model_id
        output.values_desc.kind = new_kind
        output.values_desc.unit = new_unit
        output_db.update(output.id, output)

        # check that item has really been updated in database
        updated_output = output_db.get_by_id(output.id)
        assert updated_output.id == output.id
        assert updated_output.model_id == new_model_id
        assert updated_output.module_id == output.module_id
        assert updated_output.localization == output.localization
        assert updated_output.values_desc.unit == new_unit
        assert updated_output.values_desc.kind == new_kind
        assert updated_output.values_desc.sampling == (
            output.values_desc.sampling)

        # delete an item by its ID
        output_db.remove(output.id)

        # get an item by its ID
        with pytest.raises(ItemNotFoundError):
            # it has been removed...
            output_db.get_by_id(output.id)


from fontTools.misc.loggingTools import CapturingLogHandler
from fontTools.misc.testTools import parseXML, getXML
from fontTools.misc.textTools import deHexStr
from fontTools.ttLib import TTFont, newTable
from fontTools.misc.fixedTools import log
import os
import unittest

CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
DATA_DIR = os.path.join(CURR_DIR, 'data')

VHEA_DATA_VERSION_11 = deHexStr(
    '0001 1000 '  # 1.1   version
    '01F4 '       # 500   ascent
    'FE0C '       # -500  descent
    '0000 '       # 0     lineGap
    '0BB8 '       # 3000  advanceHeightMax
    'FC16 '       # -1002 minTopSideBearing
    'FD5B '       # -677  minBottomSideBearing
    '0B70 '       # 2928  yMaxExtent
    '0000 '       # 0     caretSlopeRise
    '0001 '       # 1     caretSlopeRun
    '0000 '       # 0     caretOffset
    '0000 '       # 0     reserved1
    '0000 '       # 0     reserved2
    '0000 '       # 0     reserved3
    '0000 '       # 0     reserved4
    '0000 '       # 0     metricDataFormat
    '000C '       # 12    numberOfVMetrics
)

VHEA_DATA_VERSION_10 = deHexStr('00010000') + VHEA_DATA_VERSION_11[4:]

VHEA_VERSION_11_AS_DICT = {
    'tableTag': 'vhea',
    'tableVersion': 0x00011000,
    'ascent': 500,
    'descent': -500,
    'lineGap': 0,
    'advanceHeightMax': 3000,
    'minTopSideBearing': -1002,
    'minBottomSideBearing': -677,
    'yMaxExtent': 2928,
    'caretSlopeRise': 0,
    'caretSlopeRun': 1,
    'caretOffset': 0,
    'reserved1': 0,
    'reserved2': 0,
    'reserved3': 0,
    'reserved4': 0,
    'metricDataFormat': 0,
    'numberOfVMetrics': 12,
}

VHEA_VERSION_10_AS_DICT = dict(VHEA_VERSION_11_AS_DICT)
VHEA_VERSION_10_AS_DICT['tableVersion'] = 0x00010000

VHEA_XML_VERSION_11 = [
    '<tableVersion value="0x00011000"/>',
    '<ascent value="500"/>',
    '<descent value="-500"/>',
    '<lineGap value="0"/>',
    '<advanceHeightMax value="3000"/>',
    '<minTopSideBearing value="-1002"/>',
    '<minBottomSideBearing value="-677"/>',
    '<yMaxExtent value="2928"/>',
    '<caretSlopeRise value="0"/>',
    '<caretSlopeRun value="1"/>',
    '<caretOffset value="0"/>',
    '<reserved1 value="0"/>',
    '<reserved2 value="0"/>',
    '<reserved3 value="0"/>',
    '<reserved4 value="0"/>',
    '<metricDataFormat value="0"/>',
    '<numberOfVMetrics value="12"/>',
]

VHEA_XML_VERSION_11_AS_FLOAT = [
    '<tableVersion value="1.0625"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10 = [
    '<tableVersion value="0x00010000"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10_AS_FLOAT = [
    '<tableVersion value="1.0"/>',
] + VHEA_XML_VERSION_11[1:]

class VheaCompileOrToXMLTest(unittest.TestCase):

    def setUp(self):
        vhea = newTable('vhea')
        vhea.tableVersion = 0x00010000
        vhea.ascent = 500
        vhea.descent = -500
        vhea.lineGap = 0
        vhea.advanceHeightMax = 3000
        vhea.minTopSideBearing = -1002
        vhea.minBottomSideBearing = -677
        vhea.yMaxExtent = 2928
        vhea.caretSlopeRise = 0
        vhea.caretSlopeRun = 1
        vhea.caretOffset = 0
        vhea.metricDataFormat = 0
        vhea.numberOfVMetrics = 12
        vhea.reserved1 = vhea.reserved2 = vhea.reserved3 = vhea.reserved4 = 0
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

    def test_compile_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00010000
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_compile_version_11(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00011000
        self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))

    def test_compile_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00010000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_version_11(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00011000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)

    def test_toXML_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

class VheaDecompileOrFromXMLTest(unittest.TestCase):

    def setUp(self):
        vhea = newTable('vhea')
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

    def test_decompile_version_10(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_10, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_decompile_version_11(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_11, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_10(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_10):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_10_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_fromXML_version_11(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_11):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_11_as_float(self):
        vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_11_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

class VheaRecalcTest(unittest.TestCase):

    def test_recalc_TTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_TTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_OTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_OTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_empty(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_empty.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 0)
        self.assertEqual(vhea.minBottomSideBearing, 0)
        self.assertEqual(vhea.yMaxExtent, 0)

if __name__ == "__main__":
    import sys
    sys.exit(unittest.main())


vhea = newTable('vhea')
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

from fontTools.misc.loggingTools import CapturingLogHandler
from fontTools.misc.testTools import parseXML, getXML
from fontTools.misc.textTools import deHexStr
from fontTools.ttLib import TTFont, newTable
from fontTools.misc.fixedTools import log
import os
import unittest

CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
DATA_DIR = os.path.join(CURR_DIR, 'data')

VHEA_DATA_VERSION_11 = deHexStr(
    '0001 1000 '  # 1.1   version
    '01F4 '       # 500   ascent
    'FE0C '       # -500  descent
    '0000 '       # 0     lineGap
    '0BB8 '       # 3000  advanceHeightMax
    'FC16 '       # -1002 minTopSideBearing
    'FD5B '       # -677  minBottomSideBearing
    '0B70 '       # 2928  yMaxExtent
    '0000 '       # 0     caretSlopeRise
    '0001 '       # 1     caretSlopeRun
    '0000 '       # 0     caretOffset
    '0000 '       # 0     reserved1
    '0000 '       # 0     reserved2
    '0000 '       # 0     reserved3
    '0000 '       # 0     reserved4
    '0000 '       # 0     metricDataFormat
    '000C '       # 12    numberOfVMetrics
)

VHEA_DATA_VERSION_10 = deHexStr('00010000') + VHEA_DATA_VERSION_11[4:]

VHEA_VERSION_11_AS_DICT = {
    'tableTag': 'vhea',
    'tableVersion': 0x00011000,
    'ascent': 500,
    'descent': -500,
    'lineGap': 0,
    'advanceHeightMax': 3000,
    'minTopSideBearing': -1002,
    'minBottomSideBearing': -677,
    'yMaxExtent': 2928,
    'caretSlopeRise': 0,
    'caretSlopeRun': 1,
    'caretOffset': 0,
    'reserved1': 0,
    'reserved2': 0,
    'reserved3': 0,
    'reserved4': 0,
    'metricDataFormat': 0,
    'numberOfVMetrics': 12,
}

VHEA_VERSION_10_AS_DICT = dict(VHEA_VERSION_11_AS_DICT)
VHEA_VERSION_10_AS_DICT['tableVersion'] = 0x00010000

VHEA_XML_VERSION_11 = [
    '<tableVersion value="0x00011000"/>',
    '<ascent value="500"/>',
    '<descent value="-500"/>',
    '<lineGap value="0"/>',
    '<advanceHeightMax value="3000"/>',
    '<minTopSideBearing value="-1002"/>',
    '<minBottomSideBearing value="-677"/>',
    '<yMaxExtent value="2928"/>',
    '<caretSlopeRise value="0"/>',
    '<caretSlopeRun value="1"/>',
    '<caretOffset value="0"/>',
    '<reserved1 value="0"/>',
    '<reserved2 value="0"/>',
    '<reserved3 value="0"/>',
    '<reserved4 value="0"/>',
    '<metricDataFormat value="0"/>',
    '<numberOfVMetrics value="12"/>',
]

VHEA_XML_VERSION_11_AS_FLOAT = [
    '<tableVersion value="1.0625"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10 = [
    '<tableVersion value="0x00010000"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10_AS_FLOAT = [
    '<tableVersion value="1.0"/>',
] + VHEA_XML_VERSION_11[1:]

class VheaCompileOrToXMLTest(unittest.TestCase):

    def setUp(self):
        vhea = newTable('vhea')
        vhea.tableVersion = 0x00010000
        vhea.ascent = 500
        vhea.descent = -500
        vhea.lineGap = 0
        vhea.advanceHeightMax = 3000
        vhea.minTopSideBearing = -1002
        vhea.minBottomSideBearing = -677
        vhea.yMaxExtent = 2928
        vhea.caretSlopeRise = 0
        vhea.caretSlopeRun = 1
        vhea.caretOffset = 0
        vhea.metricDataFormat = 0
        vhea.numberOfVMetrics = 12
        vhea.reserved1 = vhea.reserved2 = vhea.reserved3 = vhea.reserved4 = 0
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

    def test_compile_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00010000
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_compile_version_11(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00011000
        self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))

    def test_compile_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00010000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_version_11(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00011000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)

    def test_toXML_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

class VheaDecompileOrFromXMLTest(unittest.TestCase):

    def setUp(self):

    def test_decompile_version_10(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_10, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_decompile_version_11(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_11, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_10(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_10):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_fromXML_version_10_as_float(self):
        vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_10_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_fromXML_version_11(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_11):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_11_as_float(self):
        vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_11_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

class VheaRecalcTest(unittest.TestCase):

    def test_recalc_TTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_TTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_OTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_OTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_empty(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_empty.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 0)
        self.assertEqual(vhea.minBottomSideBearing, 0)
        self.assertEqual(vhea.yMaxExtent, 0)

if __name__ == "__main__":
    import sys
    sys.exit(unittest.main())


from fontTools.misc.loggingTools import CapturingLogHandler
from fontTools.misc.testTools import parseXML, getXML
from fontTools.misc.textTools import deHexStr
from fontTools.ttLib import TTFont, newTable
from fontTools.misc.fixedTools import log
import os
import unittest

CURR_DIR = os.path.abspath(os.path.dirname(os.path.realpath(__file__)))
DATA_DIR = os.path.join(CURR_DIR, 'data')

VHEA_DATA_VERSION_11 = deHexStr(
    '0001 1000 '  # 1.1   version
    '01F4 '       # 500   ascent
    'FE0C '       # -500  descent
    '0000 '       # 0     lineGap
    '0BB8 '       # 3000  advanceHeightMax
    'FC16 '       # -1002 minTopSideBearing
    'FD5B '       # -677  minBottomSideBearing
    '0B70 '       # 2928  yMaxExtent
    '0000 '       # 0     caretSlopeRise
    '0001 '       # 1     caretSlopeRun
    '0000 '       # 0     caretOffset
    '0000 '       # 0     reserved1
    '0000 '       # 0     reserved2
    '0000 '       # 0     reserved3
    '0000 '       # 0     reserved4
    '0000 '       # 0     metricDataFormat
    '000C '       # 12    numberOfVMetrics
)

VHEA_DATA_VERSION_10 = deHexStr('00010000') + VHEA_DATA_VERSION_11[4:]

VHEA_VERSION_11_AS_DICT = {
    'tableTag': 'vhea',
    'tableVersion': 0x00011000,
    'ascent': 500,
    'descent': -500,
    'lineGap': 0,
    'advanceHeightMax': 3000,
    'minTopSideBearing': -1002,
    'minBottomSideBearing': -677,
    'yMaxExtent': 2928,
    'caretSlopeRise': 0,
    'caretSlopeRun': 1,
    'caretOffset': 0,
    'reserved1': 0,
    'reserved2': 0,
    'reserved3': 0,
    'reserved4': 0,
    'metricDataFormat': 0,
    'numberOfVMetrics': 12,
}

VHEA_VERSION_10_AS_DICT = dict(VHEA_VERSION_11_AS_DICT)
VHEA_VERSION_10_AS_DICT['tableVersion'] = 0x00010000

VHEA_XML_VERSION_11 = [
    '<tableVersion value="0x00011000"/>',
    '<ascent value="500"/>',
    '<descent value="-500"/>',
    '<lineGap value="0"/>',
    '<advanceHeightMax value="3000"/>',
    '<minTopSideBearing value="-1002"/>',
    '<minBottomSideBearing value="-677"/>',
    '<yMaxExtent value="2928"/>',
    '<caretSlopeRise value="0"/>',
    '<caretSlopeRun value="1"/>',
    '<caretOffset value="0"/>',
    '<reserved1 value="0"/>',
    '<reserved2 value="0"/>',
    '<reserved3 value="0"/>',
    '<reserved4 value="0"/>',
    '<metricDataFormat value="0"/>',
    '<numberOfVMetrics value="12"/>',
]

VHEA_XML_VERSION_11_AS_FLOAT = [
    '<tableVersion value="1.0625"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10 = [
    '<tableVersion value="0x00010000"/>',
] + VHEA_XML_VERSION_11[1:]

VHEA_XML_VERSION_10_AS_FLOAT = [
    '<tableVersion value="1.0"/>',
] + VHEA_XML_VERSION_11[1:]

class VheaCompileOrToXMLTest(unittest.TestCase):

    def setUp(self):
        vhea = newTable('vhea')
        vhea.tableVersion = 0x00010000
        vhea.ascent = 500
        vhea.descent = -500
        vhea.lineGap = 0
        vhea.advanceHeightMax = 3000
        vhea.minTopSideBearing = -1002
        vhea.minBottomSideBearing = -677
        vhea.yMaxExtent = 2928
        vhea.caretSlopeRise = 0
        vhea.caretSlopeRu

        vhea.caretOffset = 0
        vhea.metricDataFormat = 0
        vhea.numberOfVMetrics = 12
        vhea.reserved1 = vhea.reserved2 = vhea.reserved3 = vhea.reserved4 = 0
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

    def test_compile_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00010000
        self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))

    def test_compile_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_10, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_compile_version_11(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 0x00011000
        self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))

    def test_compile_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(VHEA_DATA_VERSION_11, vhea.compile(self.font))
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_caretOffset_as_reserved0(self):
        vhea = self.font['vhea']
        del vhea.caretOffset
        vhea.reserved0 = 0
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00010000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)

    def test_toXML_version_10_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_10)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

    def test_toXML_version_11(self):
        vhea = self.font['vhea']
        self.font['vhea'].tableVersion = 0x00011000
        self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)

    def test_toXML_version_11_as_float(self):
        vhea = self.font['vhea']
        vhea.tableVersion = 1.0625
        with CapturingLogHandler(log, "WARNING") as captor:
            self.assertEqual(getXML(vhea.toXML), VHEA_XML_VERSION_11)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)

class VheaDecompileOrFromXMLTest(unittest.TestCase):

    def setUp(self):
        vhea = newTable('vhea')
        self.font = TTFont(sfntVersion='OTTO')
        self.font['vhea'] = vhea

    def test_decompile_version_10(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_10, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_decompile_version_11(self):
        vhea = self.font['vhea']
        vhea.decompile(VHEA_DATA_VERSION_11, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_10(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_10):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_fromXML_version_10_as_float(self):
        vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_10_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_10_AS_DICT[key])

    def test_fromXML_version_11(self):
        vhea = self.font['vhea']
        for name, attrs, content in parseXML(VHEA_XML_VERSION_11):
            vhea.fromXML(name, attrs, content, self.font)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

    def test_fromXML_version_11_as_float(self):
        vhea = self.font['vhea']
        with CapturingLogHandler(log, "WARNING") as captor:
            for name, attrs, content in parseXML(VHEA_XML_VERSION_11_AS_FLOAT):
                vhea.fromXML(name, attrs, content, self.font)
        self.assertTrue(
            len([r for r in captor.records
                 if "Table version value is a float" in r.msg]) == 1)
        for key in vhea.__dict__:
            self.assertEqual(getattr(vhea, key), VHEA_VERSION_11_AS_DICT[key])

class VheaRecalcTest(unittest.TestCase):

    def test_recalc_TTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_TTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_OTF(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_OTF.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 200)
        self.assertEqual(vhea.minBottomSideBearing, 377)
        self.assertEqual(vhea.yMaxExtent, 312)

    def test_recalc_empty(self):
        font = TTFont()
        font.importXML(os.path.join(DATA_DIR, '_v_h_e_a_recalc_empty.ttx'))
        vhea = font['vhea']
        vhea.recalc(font)
        self.assertEqual(vhea.advanceHeightMax, 900)
        self.assertEqual(vhea.minTopSideBearing, 0)
        self.assertEqual(vhea.minBottomSideBearing, 0)
        self.assertEqual(vhea.yMaxExtent, 0)

if __name__ == "__main__":
    import sys
    sys.exit(unittest.main())


# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize

class ParserGenerator(object):
    def __init__(self, filename, stream=None):
        close_stream = None
        if stream is None:
            stream = open(filename)
            close_stream = stream.close
        self.filename = filename
        self.stream = stream
        self.generator = tokenize.generate_tokens(stream.readline)
        self.gettoken()  # Initialize lookahead
        self.dfas, self.startsymbol = self.parse()
        if close_stream is not None:
            close_stream()
        self.first = {}  # map from symbol name to set of tokens
        self.addfirstsets()

    def make_grammar(self):
        c = grammar.Grammar()
        names = list(self.dfas.keys())
        names.sort()
        names.remove(self.startsymbol)
        names.insert(0, self.startsymbol)
        for name in names:
            i = 256 + len(c.symbol2number)
            c.symbol2number[name] = i
            c.number2symbol[i] = name
        for name in names:
            dfa = self.dfas[name]
            states = []
            for state in dfa:
                arcs = []
                for label, next in state.arcs.items():
                    arcs.append((self.make_label(c, label), dfa.index(next)))
                if state.isfinal:
                    arcs.append((0, dfa.index(state)))
                states.append(arcs)
            c.states.append(states)
            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
        c.start = c.symbol2number[self.startsymbol]
        return c

    def make_first(self, c, name):
        rawfirst = self.first[name]
        first = {}
        for label in rawfirst:
            ilabel = self.make_label(c, label)
            ##assert ilabel not in first # XXX failed on <> ... !=
            first[ilabel] = 1
        return first

    def make_label(self, c, label):
        # XXX Maybe this should be a method on a subclass of converter?
        ilabel = len(c.labels)
        if label[0].isalpha():
            # Either a symbol name or a named token
            if label in c.symbol2number:
                # A symbol name (a non-terminal)
                if label in c.symbol2label:
                    return c.symbol2label[label]
                else:
                    c.labels.append((c.symbol2number[label], None))
                    c.symbol2label[label] = ilabel
                    return ilabel
            else:
                # A named token (NAME, NUMBER, STRING)
                itoken = getattr(token, label, None)
                assert isinstance(itoken, int), label
                assert itoken in token.tok_name, label
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel
        else:
            # Either a keyword or an operator
            assert label[0] in ('"', "'"), label
            value = eval(label)
            if value[0].isalpha():
                # A keyword
                if value in c.keywords:
                    return c.keywords[value]
                else:
                    c.labels.append((token.NAME, value))
                    c.keywords[value] = ilabel
                    return ilabel
            else:
                # An operator (any non-numeric token)
                itoken = token.opmap[value]  # Fails if unknown token
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel

    def addfirstsets(self):
        names = list(self.dfas.keys())
        names.sort()
        for name in names:
            if name not in self.first:
                self.calcfirst(name)

    def calcfirst(self, name):
        dfa = self.dfas[name]
        self.first[name] = None  # dummy to detect left recursion
        state = dfa[0]
        totalset = {}
        overlapcheck = {}
        for label, next in state.arcs.items():
            if label in self.dfas:
                if label in self.first:
                    fset = self.first[label]
                    if fset is None:
                        raise ValueError("recursion for rule %r" % name)
                else:
                    self.calcfirst(label)
                    fset = self.first[label]
                totalset.update(fset)
                overlapcheck[label] = fset
            else:
                totalset[label] = 1
                overlapcheck[label] = {label: 1}
        inverse = {}
        for label, itsfirst in overlapcheck.items():
            for symbol in itsfirst:
                if symbol in inverse:
                    raise ValueError("rule %s is ambiguous; %s is in the"
                                     " first sets of %s as well as %s" %
                                     (name, symbol, label, inverse[symbol]))
                inverse[symbol] = label
        self.first[name] = totalset

    def parse(self):
        dfas = {}
        startsymbol = None
        while self.type != token.ENDMARKER:
            while self.type == token.NEWLINE:
                self.gettoken()
            name = self.expect(token.NAME)
            self.expect(token.OP, ":")
            a, z = self.parse_rhs()
            self.expect(token.NEWLINE)
            dfa = self.make_dfa(a, z)
            self.simplify_dfa(dfa)
            dfas[name] = dfa
            #print name, oldlen, newlen
            if startsymbol is None:
                startsymbol = name
        return dfas, startsymbol

    def make_dfa(self, start, finish):
        # To turn an NFA into a DFA, we define the states of the DFA
        # to correspond to *sets* of states of the NFA.  Then do some
        # state reduction.  Let's represent sets as dicts with 1 for
        assert isinstance(start, NFAState)
        assert isinstance(finish, NFAState)

        def closure(state):
            base = {}
            addclosure(state, base)
            return base

        def addclosure(state, base):
            assert isinstance(state, NFAState)
            if state in base:
                return
            base[state] = 1
            for label, next in state.arcs:
                if label is None:
                    addclosure(next, base)

        states = [DFAState(closure(start), finish)]
        for state in states:  # NB states grows while we're iterating
            arcs = {}
            for nfastate in state.nfaset:
                for label, next in nfastate.arcs:
                    if label is not None:
                        addclosure(next, arcs.setdefault(label, {}))
            for label, nfaset in arcs.items():
                for st in states:
                    if st.nfaset == nfaset:
                        break
                else:
                    st = DFAState(nfaset, finish)
                    states.append(st)
                state.addarc(st, label)
        return states  # List of DFAState instances; first one is start

    def dump_nfa(self, name, start, finish):
        print("Dump of NFA for", name)
        todo = [start]
        for i, state in enumerate(todo):
            print("  State", i, state is finish and "(final)" or "")
            for label, next in state.arcs:
                if next in todo:
                    j = todo.index(next)
                else:
                    j = len(todo)
                    todo.append(next)
                if label is None:
                    print("    -> %d" % j)
                else:
                    print("    %s -> %d" % (label, j))

    def dump_dfa(self, name, dfa):
        print("Dump of DFA for", name)
        for i, state in enumerate(dfa):
            print("  State", i, state.isfinal and "(final)" or "")
            for label, next in state.arcs.items():
                print("    %s -> %d" % (label, dfa.index(next)))

    def simplify_dfa(self, dfa):
        # This is not theoretically optimal, but works well enough.
        # Algorithm: repeatedly look for two states that have the same
        # set of arcs (same labels pointing to the same nodes) and
        # unify them, until things stop changing.

        # dfa is a list of DFAState instances
        changes = True
        while changes:
            changes = False
            for i, state_i in enumerate(dfa):
                for j in range(i + 1, len(dfa)):
                    state_j = dfa[j]
                    if state_i == state_j:
                        del dfa[j]
                        for state in dfa:
                            state.unifystate(state_j, state_i)
                        changes = True
                        break

    def parse_rhs(self):
        a, z = self.parse_alt()
        if self.value != "|":
            return a, z
        else:
            aa = NFAState()
            zz = NFAState()
            aa.addarc(a)
            z.addarc(zz)
            while self.value == "|":
                self.gettoken()
                a, z = self.parse_alt()
                aa.addarc(a)
                z.addarc(zz)
            return aa, zz

    def parse_alt(self):
        a, b = self.parse_item()
        while (self.value in ("(", "[") or
               self.type in (token.NAME, token.STRING)):
            c, d = self.parse_item()
            b.addarc(c)
            b = d
        return a, b

if self.value == "[":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, "]")
            a.addarc(z)
            return a, z
        else:
            a, z = self.parse_atom()
            value = self.value
            if value not in ("+", "*"):
                return a, z
            self.gettoken()
            z.addarc(a)
            if value == "+":
                return a, z
            else:
                return a, a

    def parse_atom(self):
        if self.value == "(":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, ")")
            return a, z
        elif self.type in (token.NAME, token.STRING):
            a = NFAState()
            z = NFAState()
            a.addarc(z, self.value)
            self.gettoken()
            return a, z
        else:
            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
                             self.type, self.value)

    def expect(self, type, value=None):
        if self.type != type or (value is not None and self.value != value):
            self.raise_error("expected %s/%s, got %s/%s",
                             type, value, self.type, self.value)
        value = self.value
        self.gettoken()
        return value

    def gettoken(self):
        tup = next(self.generator)
        while tup[0] in (token.COMMENT, token.NL):
            tup = next(self.generator)
        self.type, self.value, self.begin, prefix = tup

    def raise_error(self, msg, *args):
        if args:
            try:
                msg = msg % args
            except:
                msg = " ".join([msg] + list(map(str, args)))
        line = open(self.filename).readlines()[self.begin[0]]
        raise SyntaxError(msg, (self.filename, self.begin[0],
                                self.begin[1], line))

class NFAState(object):
    def __init__(self):
        self.arcs = []  # list of (label, NFAState) pairs

    def addarc(self, next, label=None):
        assert label is None or isinstance(label, str)
        assert isinstance(next, NFAState)
        self.arcs.append((label, next))

class DFAState(object):
    def __init__(self, nfaset, final):
        assert isinstance(nfaset, dict)
        assert isinstance(next(iter(nfaset)), NFAState)
        assert isinstance(final, NFAState)
        self.nfaset = nfaset
        self.isfinal = final in nfaset
        self.arcs = {}  # map from label to DFAState

    def addarc(self, next, label):
        assert isinstance(label, str)
        assert label not in self.arcs
        assert isinstance(next, DFAState)
        self.arcs[label] = next

    def unifystate(self, old, new):
        for label, next in self.arcs.items():
            if next is old:
                self.arcs[label] = new

    def __eq__(self, other):
        # Equality test -- ignore the nfaset instance variable
        assert isinstance(other, DFAState)
        if self.isfinal != other.isfinal:
            return False
        # Can't just return self.arcs == other.arcs, because that
        # would invoke this method recursively, with cycles...
        if len(self.arcs) != len(other.arcs):
            return False
        for label, next in self.arcs.items():
            if next is not other.arcs.get(label):
                return False
        return True

    __hash__ = None  # For Py3 compatibility.

def generate_grammar(filename="Grammar.txt"):
    p = ParserGenerator(filename)
    return p.make_grammar()


assert isinstance(label, str)
        assert label not in self.arcs
        assert isinstance(next, DFAState)
        self.arcs[label] = next

# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize

class ParserGenerator(object):
    def __init__(self, filename, stream=None):
        close_stream = None
        if stream is None:
            stream = open(filename)
            close_stream = stream.close
        self.filename = filename
        self.stream = stream
        self.generator = tokenize.generate_tokens(stream.readline)
        self.gettoken()  # Initialize lookahead
        self.dfas, self.startsymbol = self.parse()
        if close_stream is not None:
            close_stream()
        self.first = {}  # map from symbol name to set of tokens
        self.addfirstsets()

    def make_grammar(self):
        c = grammar.Grammar()
        names = list(self.dfas.keys())
        names.sort()
        names.remove(self.startsymbol)
        names.insert(0, self.startsymbol)
        for name in names:
            i = 256 + len(c.symbol2number)
            c.symbol2number[name] = i
            c.number2symbol[i] = name
        for name in names:
            dfa = self.dfas[name]
            states = []
            for state in dfa:
                arcs = []
                for label, next in state.arcs.items():
                    arcs.append((self.make_label(c, label), dfa.index(next)))
                if state.isfinal:
                    arcs.append((0, dfa.index(state)))
                states.append(arcs)
            c.states.append(states)
            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
        c.start = c.symbol2number[self.startsymbol]
        return c

    def make_first(self, c, name):
        rawfirst = self.first[name]
        first = {}
        for label in rawfirst:
            ilabel = self.make_label(c, label)
            ##assert ilabel not in first # XXX failed on <> ... !=
            first[ilabel] = 1
        return first

    def make_label(self, c, label):
        # XXX Maybe this should be a method on a subclass of converter?
        ilabel = len(c.labels)
        if label[0].isalpha():
            # Either a symbol name or a named token
            if label in c.symbol2number:
                # A symbol name (a non-terminal)
                if label in c.symbol2label:
                    return c.symbol2label[label]
                else:
                    c.labels.append((c.symbol2number[label], None))
                    c.symbol2label[label] = ilabel
                    return ilabel
            else:
                # A named token (NAME, NUMBER, STRING)
                itoken = getattr(token, label, None)
                assert isinstance(itoken, int), label
                assert itoken in token.tok_name, label
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel
        else:
            # Either a keyword or an operator
            assert label[0] in ('"', "'"), label
            value = eval(label)
            if value[0].isalpha():
                # A keyword
                if value in c.keywords:
                    return c.keywords[value]
                else:
                    c.labels.append((token.NAME, value))
                    c.keywords[value] = ilabel
                    return ilabel
            else:
                # An operator (any non-numeric token)
                itoken = token.opmap[value]  # Fails if unknown token
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel

    def addfirstsets(self):
        names = list(self.dfas.keys())
        names.sort()
        for name in names:
            if name not in self.first:
                self.calcfirst(name)

    def calcfirst(self, name):
        dfa = self.dfas[name]
        self.first[name] = None  # dummy to detect left recursion
        state = dfa[0]
        totalset = {}
        overlapcheck = {}
        for label, next in state.arcs.items():
            if label in self.dfas:
                if label in self.first:
                    fset = self.first[label]
                    if fset is None:
                        raise ValueError("recursion for rule %r" % name)
                else:
                    self.calcfirst(label)
                    fset = self.first[label]
                totalset.update(fset)
                overlapcheck[label] = fset
            else:
                totalset[label] = 1
                overlapcheck[label] = {label: 1}
        inverse = {}
        for label, itsfirst in overlapcheck.items():
            for symbol in itsfirst:
                if symbol in inverse:
                    raise ValueError("rule %s is ambiguous; %s is in the"
                                     " first sets of %s as well as %s" %
                                     (name, symbol, label, inverse[symbol]))
                inverse[symbol] = label
        self.first[name] = totalset

    def parse(self):
        dfas = {}
        startsymbol = None
        while self.type != token.ENDMARKER:
            while self.type == token.NEWLINE:
                self.gettoken()
            name = self.expect(token.NAME)
            self.expect(token.OP, ":")
            a, z = self.parse_rhs()
            self.expect(token.NEWLINE)
            dfa = self.make_dfa(a, z)
            self.simplify_dfa(dfa)
            dfas[name] = dfa
            #print name, oldlen, newlen
            if startsymbol is None:
                startsymbol = name
        return dfas, startsymbol

    def make_dfa(self, start, finish):
        # To turn an NFA into a DFA, we define the states of the DFA
        # to correspond to *sets* of states of the NFA.  Then do some
        # state reduction.  Let's represent sets as dicts with 1 for
        assert isinstance(start, NFAState)
        assert isinstance(finish, NFAState)

        def closure(state):
            base = {}
            addclosure(state, base)
            return base

        def addclosure(state, base):
            assert isinstance(state, NFAState)
            if state in base:
                return
            base[state] = 1
            for label, next in state.arcs:
                if label is None:
                    addclosure(next, base)

        states = [DFAState(closure(start), finish)]
        for state in states:  # NB states grows while we're iterating
            arcs = {}
            for nfastate in state.nfaset:
                for label, next in nfastate.arcs:
                    if label is not None:
                        addclosure(next, arcs.setdefault(label, {}))
            for label, nfaset in arcs.items():
                for st in states:
                    if st.nfaset == nfaset:
                        break
                else:
                    st = DFAState(nfaset, finish)
                    states.append(st)
                state.addarc(st, label)
        return states  # List of DFAState instances; first one is start

    def dump_nfa(self, name, start, finish):
        print("Dump of NFA for", name)
        todo = [start]
        for i, state in enumerate(todo):
            print("  State", i, state is finish and "(final)" or "")
            for label, next in state.arcs:
                if next in todo:
                    j = todo.index(next)
                else:
                    j = len(todo)
                    todo.append(next)
                if label is None:
                    print("    -> %d" % j)
                else:
                    print("    %s -> %d" % (label, j))

    def dump_dfa(self, name, dfa):
        print("Dump of DFA for", name)
        for i, state in enumerate(dfa):
            print("  State", i, state.isfinal and "(final)" or "")
            for label, next in state.arcs.items():
                print("    %s -> %d" % (label, dfa.index(next)))

    def simplify_dfa(self, dfa):
        # This is not theoretically optimal, but works well enough.
        # Algorithm: repeatedly look for two states that have the same
        # set of arcs (same labels pointing to the same nodes) and
        # unify them, until things stop changing.

        # dfa is a list of DFAState instances
        changes = True
        while changes:
            changes = False
            for i, state_i in enumerate(dfa):
                for j in range(i + 1, len(dfa)):
                    state_j = dfa[j]
                    if state_i == state_j:
                        del dfa[j]
                        for state in dfa:
                            state.unifystate(state_j, state_i)
                        changes = True
                        break

    def parse_rhs(self):
        a, z = self.parse_alt()
        if self.value != "|":
            return a, z
        else:
            aa = NFAState()
            zz = NFAState()
            aa.addarc(a)
            z.addarc(zz)
            while self.value == "|":
                self.gettoken()
                a, z = self.parse_alt()
                aa.addarc(a)
                z.addarc(zz)
            return aa, zz

    def parse_alt(self):
        a, b = self.parse_item()
        while (self.value in ("(", "[") or
               self.type in (token.NAME, token.STRING)):
            c, d = self.parse_item()
            b.addarc(c)
            b = d
        return a, b

    def parse_item(self):
        if self.value == "[":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, "]")
            a.addarc(z)
            return a, z
        else:
            a, z = self.parse_atom()
            value = self.value
            if value not in ("+", "*"):
                return a, z
            self.gettoken()
            z.addarc(a)
            if value == "+":
                return a, z
            else:
                return a, a

    def parse_atom(self):
        if self.value == "(":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, ")")
            return a, z
        elif self.type in (token.NAME, token.STRING):
            a = NFAState()
            z = NFAState()
            a.addarc(z, self.value)
            self.gettoken()
            return a, z
        else:
            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
                             self.type, self.value)

    def expect(self, type, value=None):
        if self.type != type or (value is not None and self.value != value):
            self.raise_error("expected %s/%s, got %s/%s",
                             type, value, self.type, self.value)
        value = self.value
        self.gettoken()
        return value

    def gettoken(self):
        tup = next(self.generator)
        while tup[0] in (token.COMMENT, token.NL):
            tup = next(self.generator)
        self.type, self.value, self.begin, prefix = tup

    def raise_error(self, msg, *args):
        if args:
            try:
                msg = msg % args
            except:
                msg = " ".join([msg] + list(map(str, args)))
        line = open(self.filename).readlines()[self.begin[0]]
        raise SyntaxError(msg, (self.filename, self.begin[0],
                                self.begin[1], line))

class NFAState(object):
    def __init__(self):
        self.arcs = []  # list of (label, NFAState) pairs

    def addarc(self, next, label=None):
        assert label is None or isinstance(label, str)
        assert isinstance(next, NFAState)
        self.arcs.append((label, next))

class DFAState(object):
    def __init__(self, nfaset, final):
        assert isinstance(nfaset, dict)
        assert isinstance(next(iter(nfaset)), NFAState)
        assert isinstance(final, NFAState)
        self.nfaset = nfaset
        self.isfinal = final in nfaset
        self.arcs = {}  # map from label to DFAState

    def addarc(self, next, label):

    def unifystate(self, old, new):
        for label, next in self.arcs.items():
            if next is old:
                self.arcs[label] = new

    def __eq__(self, other):
        # Equality test -- ignore the nfaset instance variable
        assert isinstance(other, DFAState)
        if self.isfinal != other.isfinal:
            return False
        # Can't just return self.arcs == other.arcs, because that
        # would invoke this method recursively, with cycles...
        if len(self.arcs) != len(other.arcs):
            return False
        for label, next in self.arcs.items():
            if next is not other.arcs.get(label):
                return False
        return True

    __hash__ = None  # For Py3 compatibility.

def generate_grammar(filename="Grammar.txt"):
    p = ParserGenerator(filename)
    return p.make_grammar()


# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize

class ParserGenerator(object):
    def __init__(self, filename, stream=None):
        close_stream = None
        if stream is None:
            stream = open(filename)
            close_stream = stream.close
        self.filename = filename
        self.stream = stream
        self.generator = tokenize.generate_tokens(stream.readline)
        self.gettoken()  # Initialize lookahead
        self.dfas, self.startsymbol = self.parse()
        if close_stream is not None:
            close_stream()
        self.first = {}  # map from symbol name to set of tokens
        self.addfirstsets()

    def make_grammar(self):
        c = grammar.Grammar()
        names = list(self.dfas.keys())
        names.sort()
        names.remove(self.startsymbol)
        names.insert(0, self.startsymbol)
        for name in names:
            i = 256 + len(c.symbol2number)
            c.symbol2number[name] = i
            c.number2symbol[i] = name
        for name in names:
            dfa = self.dfas[name]
            states = []
            for state in dfa:
                arcs = []
                for label, next in state.arcs.items():
                    arcs.append((self.make_label(c, label), dfa.index(next)))
                if state.isfinal:
                    arcs.append((0, dfa.index(state)))
                states.append(arcs)
            c.states.append(states)
            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
        c.start = c.symbol2number[self.startsymbol]
        return c

    def make_first(self, c, name):
        rawfirst = self.first[name]
        first = {}
        for label in rawfirst:
            ilabel = self.make_label(c, label)
            ##assert ilabel not in first # XXX failed on <> ... !=
            first[ilabel] = 1
        return first

    def make_label(self, c, label):
        # XXX Maybe this should be a method on a subclass of converter?
        ilabel = len(c.labels)
        if label[0].isalpha():
            # Either a symbol name or a named token
            if label in c.symbol2number:
                # A symbol name (a non-terminal)
                if label in c.symbol2label:
                    return c.symbol2label[label]
                else:
                    c.labels.append((c.symbol2number[label], None))
                    c.symbol2label[label] = ilabel
                    return ilabel
            else:
                # A named token (NAME, NUMBER, STRING)
                itoken = getattr(token, label, None)
                assert isinstance(itoken, int), label
                assert itoken in token.tok_name, label
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel
        else:
            # Either a keyword or an operator
            assert label[0] in ('"', "'"), label
            value = eval(label)
            if value[0].isalpha():
                # A keyword
                if value in c.keywords:
                    return c.keywords[value]
                else:
                    c.labels.append((token.NAME, value))
                    c.keywords[value] = ilabel
                    return ilabel
            else:
                # An operator (any non-numeric token)
                itoken = token.opmap[value]  # Fails if unknown token
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel

    def addfirstsets(self):
        names = list(self.dfas.keys())
        names.sort()
        for name in names:
            if name not in self.first:
                self.calcfirst(name)

    def calcfirst(self, name):
        dfa = self.dfas[name]
        self.first[name] = None  # dummy to detect left recursion
        state = dfa[0]
        totalset = {}
        overlapcheck = {}
        for label, next in state.arcs.items():
            if label in self.dfas:
                if label in self.first:
                    fset = self.first[label]
                    if fset is None:
                        raise ValueError("recursion for rule %r" % name)
                else:
                    self.calcfirst(label)
                    fset = self.first[label]
                totalset.update(fset)
                overlapcheck[label] = fset
            else:
                totalset[label] = 1
                overlapcheck[label] = {label: 1}
        inverse = {}
        for label, itsfirst in overlapcheck.items():
            for symbol in itsfirst:
                if symbol in inverse:
                    raise ValueError("rule %s is ambiguous; %s is in the"
                                     " first sets of %s as well as %s" %
                                     (name, symbol, label, inverse[symbol]))
                inverse[symbol] = label
        self.first[name] = totalset

    def parse(self):
        dfas = {}
        startsymbol = None
        while self.type != token.ENDMARKER:
            while self.type == token.NEWLINE:
                self.gettoken()
            name = self.expect(token.NAME)
            self.expect(token.OP, ":")
            a, z = self.parse_rhs()
            self.expect(token.NEWLINE)
            dfa = self.make_dfa(a, z)
            self.simplify_dfa(dfa)
            dfas[name] = dfa
            #print name, oldlen, newlen
            if startsymbol is None:
                startsymbol = name
        return dfas, startsymbol

    def make_dfa(self, start, finish):
        # To turn an NFA into a DFA, we define the states of the DFA
        # to correspond to *sets* of states of the NFA.  Then do some
        # state reduction.  Let's represent sets as dicts with 1 for
        assert isinstance(start, NFAState)
        assert isinstance(finish, NFAState)

        def closure(state):
            base = {}
            addclosure(state, base)
            return base

        def addclosure(state, base):
            assert isinstance(state, NFAState)
            if state in base:
                return
            base[state] = 1
            for label, next in state.arcs:
                if label is None:
                    addclosure(next, base)

        states = [DFAState(closure(start), finish)]
        for state in states:  # NB states grows while we're iterating
            arcs = {}
            for nfastate in state.nfaset:
                for label, next in nfastate.arcs:
                    if label is not None:
                        addclosure(next, arcs.setdefault(label, {}))
            for label, nfaset in arcs.items():
                for st in states:
                    if st.nfaset == nfaset:
                        break
                else:
                    st = DFAState(nfaset, finish)
                    states.append(st)
                state.addarc(st, label)
        return states  # List of DFAState instances; first one is start

    def dump_nfa(self, name, start, finish):
        print("Dump of NFA for", name)
        todo = [start]
        for i, state in enumerate(todo):
            print("  State", i, state is finish and "(final)" or "")
            for label, next in state.arcs:
                if next in todo:
                    j = todo.index(next)
                else:
                    j = len(todo)
                    todo.append(next)
                if label is None:
                    print("    -> %d" % j)
                else:
                    print("    %s -> %d" % (label, j))

    def dump_dfa(self, name, dfa):
        print("Dump of DFA for", name)
        for i, state in enumerate(dfa):
            print("  State", i, state.isfinal and "(final)" or "")
            for label, next in state.arcs.items():
                print("    %s -> %d" % (label, dfa.index(next)))

    def simplify_dfa(self, dfa):
        # This is not theoretically optimal, but works well enough.
        # Algorithm: repeatedly look for two states that have the same
        # set of arcs (same labels pointing to the same nodes) and
        # unify them, until things stop changing.

        # dfa is a list of DFAState instances
        changes = True
        while changes:
            changes = False
            for i, state_i in enumerate(dfa):
                for j in range(i + 1, len(dfa)):
                    state_j = dfa[j]
                    if state_i == state_j:
                        del dfa[j]
                        for state in dfa:
                            state.unifystate(state_j, state_i)
                        changes = True
                        break

    def parse_rhs(self):
        a, z = self.parse_alt()
        if self.value != "|":
            return a, z
        else:
            aa = NFAState()
            zz = NFAState()
            aa.addarc(a)
            z.addarc(zz)
            while self.value == "|":
                self.gettoken()
                a, z = self.parse_alt()
                aa.addarc(a)
                z.addarc(zz)
            return aa, zz

    def parse_alt(self):
        a, b = self.parse_item()
        while (self.value in ("(", "[") or
               self.type in (token.NAME, token.STRING)):
            c, d = self.parse_item()
            b.addarc(c)
            b = d
        return a, b

    def parse_item(self):
        if self.value == "[":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, "]")
            a.addarc(z)
            return a, z
        else:
            a, z = self.parse_atom()
            value = self.value
            if value not in ("+", "*"):
                return a, z
            self.gettoken()
            z.addarc(a)
            if value == "+":
                return a, z
            else:
                return a, a

    def parse_atom(self):
        if self.value == "(":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, ")")
            return a, z
        elif self.type in (token.NAME, token.STRING):
            a = NFAState()
            z = NFAState()
            a.addarc(z, self.value)
            self.gettoken()
            return a, z
        else:
            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
                             self.type, self.value)

    def expect(self, type, value=None):
        if self.type != type or (value is not None and self.value != value):
            self.raise_error("expected %s/%s, got %s/%s",
                             type, value, self.type, self.value)
        value = self.value
        self.gettoken()
        return value

    def gettoken(self):
        tup = next(self.generator)
        while tup[0] in (token.COMMENT, token.NL):
            tup = next(self.generator)
        self.type, self.value, self.begin, prefix = tup

    def raise_error(self, msg, *args):
        if args:
            try:
                msg = msg % args
            except:
                msg = " ".join([msg] + list(map(str, args)))
        line = open(self.filename).readlines()[self.begin[0]]
        raise SyntaxError(msg, (self.filename, self.begin[0],
                                self.begin[1], line))

class NFAState(object):
    def __init__(self):
        self.arcs = []  # list of (label, NFAState) pairs

    def addarc(self, next, label=None):
        assert label is None or isinstance(label, str)
        assert isinstance(next, NFAState)
        self.arcs.append((label, next))

class DFAState(object):
    def __init__(self, nfaset, final):
        assert isinstance(nfaset, dict)
        assert isinstance(next(iter(nfaset)), NFAState)
        assert isinstance(final, NFAState)
        self.nfaset = nfaset
        self.isfinal = final in nfaset
        self.arcs = {}  # map from label to DFAState

    def addarc(self, next, label):
        assert isinstance(label, str)
        assert label not in self.arcs
        assert isinstance(next, DFAState)
        self.arcs[label] = next

    def unifystate(self, old, new):
        for label, next in self.arcs.items():
            if next is old:
                self.arcs[label] = new

    def __eq__(self, other):
        # Equality test -- ignore the nfaset instance variable
        assert isinstance(other, DF

        if self.isfinal != other.isfinal:
            return False
        # Can't just return self.arcs == other.arcs, because that
        # would invoke this method recursively, with cycles...
        if len(self.arcs) != len(other.arcs):
            return False
        for label, next in self.arcs.items():
            if next is not other.arcs.get(label):
                return False
        return True

    __hash__ = None  # For Py3 compatibility.

def generate_grammar(filename="Grammar.txt"):
    p = ParserGenerator(filename)
    return p.make_grammar()


while (self.value in ("(", "[") or
               self.type in (token.NAME, token.STRING)):
            c, d = self.parse_item()
            b.addarc(c)
            b = d

# Pgen imports
from . import grammar
from jedi.parser import token
from jedi.parser import tokenize

class ParserGenerator(object):
    def __init__(self, filename, stream=None):
        close_stream = None
        if stream is None:
            stream = open(filename)
            close_stream = stream.close
        self.filename = filename
        self.stream = stream
        self.generator = tokenize.generate_tokens(stream.readline)
        self.gettoken()  # Initialize lookahead
        self.dfas, self.startsymbol = self.parse()
        if close_stream is not None:
            close_stream()
        self.first = {}  # map from symbol name to set of tokens
        self.addfirstsets()

    def make_grammar(self):
        c = grammar.Grammar()
        names = list(self.dfas.keys())
        names.sort()
        names.remove(self.startsymbol)
        names.insert(0, self.startsymbol)
        for name in names:
            i = 256 + len(c.symbol2number)
            c.symbol2number[name] = i
            c.number2symbol[i] = name
        for name in names:
            dfa = self.dfas[name]
            states = []
            for state in dfa:
                arcs = []
                for label, next in state.arcs.items():
                    arcs.append((self.make_label(c, label), dfa.index(next)))
                if state.isfinal:
                    arcs.append((0, dfa.index(state)))
                states.append(arcs)
            c.states.append(states)
            c.dfas[c.symbol2number[name]] = (states, self.make_first(c, name))
        c.start = c.symbol2number[self.startsymbol]
        return c

    def make_first(self, c, name):
        rawfirst = self.first[name]
        first = {}
        for label in rawfirst:
            ilabel = self.make_label(c, label)
            ##assert ilabel not in first # XXX failed on <> ... !=
            first[ilabel] = 1
        return first

    def make_label(self, c, label):
        # XXX Maybe this should be a method on a subclass of converter?
        ilabel = len(c.labels)
        if label[0].isalpha():
            # Either a symbol name or a named token
            if label in c.symbol2number:
                # A symbol name (a non-terminal)
                if label in c.symbol2label:
                    return c.symbol2label[label]
                else:
                    c.labels.append((c.symbol2number[label], None))
                    c.symbol2label[label] = ilabel
                    return ilabel
            else:
                # A named token (NAME, NUMBER, STRING)
                itoken = getattr(token, label, None)
                assert isinstance(itoken, int), label
                assert itoken in token.tok_name, label
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel
        else:
            # Either a keyword or an operator
            assert label[0] in ('"', "'"), label
            value = eval(label)
            if value[0].isalpha():
                # A keyword
                if value in c.keywords:
                    return c.keywords[value]
                else:
                    c.labels.append((token.NAME, value))
                    c.keywords[value] = ilabel
                    return ilabel
            else:
                # An operator (any non-numeric token)
                itoken = token.opmap[value]  # Fails if unknown token
                if itoken in c.tokens:
                    return c.tokens[itoken]
                else:
                    c.labels.append((itoken, None))
                    c.tokens[itoken] = ilabel
                    return ilabel

    def addfirstsets(self):
        names = list(self.dfas.keys())
        names.sort()
        for name in names:
            if name not in self.first:
                self.calcfirst(name)

    def calcfirst(self, name):
        dfa = self.dfas[name]
        self.first[name] = None  # dummy to detect left recursion
        state = dfa[0]
        totalset = {}
        overlapcheck = {}
        for label, next in state.arcs.items():
            if label in self.dfas:
                if label in self.first:
                    fset = self.first[label]
                    if fset is None:
                        raise ValueError("recursion for rule %r" % name)
                else:
                    self.calcfirst(label)
                    fset = self.first[label]
                totalset.update(fset)
                overlapcheck[label] = fset
            else:
                totalset[label] = 1
                overlapcheck[label] = {label: 1}
        inverse = {}
        for label, itsfirst in overlapcheck.items():
            for symbol in itsfirst:
                if symbol in inverse:
                    raise ValueError("rule %s is ambiguous; %s is in the"
                                     " first sets of %s as well as %s" %
                                     (name, symbol, label, inverse[symbol]))
                inverse[symbol] = label
        self.first[name] = totalset

    def parse(self):
        dfas = {}
        startsymbol = None
        while self.type != token.ENDMARKER:
            while self.type == token.NEWLINE:
                self.gettoken()
            name = self.expect(token.NAME)
            self.expect(token.OP, ":")
            a, z = self.parse_rhs()
            self.expect(token.NEWLINE)
            dfa = self.make_dfa(a, z)
            self.simplify_dfa(dfa)
            dfas[name] = dfa
            #print name, oldlen, newlen
            if startsymbol is None:
                startsymbol = name
        return dfas, startsymbol

    def make_dfa(self, start, finish):
        # To turn an NFA into a DFA, we define the states of the DFA
        # to correspond to *sets* of states of the NFA.  Then do some
        # state reduction.  Let's represent sets as dicts with 1 for
        assert isinstance(start, NFAState)
        assert isinstance(finish, NFAState)

        def closure(state):
            base = {}
            addclosure(state, base)
            return base

        def addclosure(state, base):
            assert isinstance(state, NFAState)
            if state in base:
                return
            base[state] = 1
            for label, next in state.arcs:
                if label is None:
                    addclosure(next, base)

        states = [DFAState(closure(start), finish)]
        for state in states:  # NB states grows while we're iterating
            arcs = {}
            for nfastate in state.nfaset:
                for label, next in nfastate.arcs:
                    if label is not None:
                        addclosure(next, arcs.setdefault(label, {}))
            for label, nfaset in arcs.items():
                for st in states:
                    if st.nfaset == nfaset:
                        break
                else:
                    st = DFAState(nfaset, finish)
                    states.append(st)
                state.addarc(st, label)
        return states  # List of DFAState instances; first one is start

    def dump_nfa(self, name, start, finish):
        print("Dump of NFA for", name)
        todo = [start]
        for i, state in enumerate(todo):
            print("  State", i, state is finish and "(final)" or "")
            for label, next in state.arcs:
                if next in todo:
                    j = todo.index(next)
                else:
                    j = len(todo)
                    todo.append(next)
                if label is None:
                    print("    -> %d" % j)
                else:
                    print("    %s -> %d" % (label, j))

    def dump_dfa(self, name, dfa):
        print("Dump of DFA for", name)
        for i, state in enumerate(dfa):
            print("  State", i, state.isfinal and "(final)" or "")
            for label, next in state.arcs.items():
                print("    %s -> %d" % (label, dfa.index(next)))

    def simplify_dfa(self, dfa):
        # This is not theoretically optimal, but works well enough.
        # Algorithm: repeatedly look for two states that have the same
        # set of arcs (same labels pointing to the same nodes) and
        # unify them, until things stop changing.

        # dfa is a list of DFAState instances
        changes = True
        while changes:
            changes = False
            for i, state_i in enumerate(dfa):
                for j in range(i + 1, len(dfa)):
                    state_j = dfa[j]
                    if state_i == state_j:
                        del dfa[j]
                        for state in dfa:
                            state.unifystate(state_j, state_i)
                        changes = True
                        break

    def parse_rhs(self):
        a, z = self.parse_alt()
        if self.value != "|":
            return a, z
        else:
            aa = NFAState()
            zz = NFAState()
            aa.addarc(a)
            z.addarc(zz)
            while self.value == "|":
                self.gettoken()
                a, z = self.parse_alt()
                aa.addarc(a)
                z.addarc(zz)
            return aa, zz

    def parse_alt(self):
        a, b = self.parse_item()

        return a, b

    def parse_item(self):
        if self.value == "[":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, "]")
            a.addarc(z)
            return a, z
        else:
            a, z = self.parse_atom()
            value = self.value
            if value not in ("+", "*"):
                return a, z
            self.gettoken()
            z.addarc(a)
            if value == "+":
                return a, z
            else:
                return a, a

    def parse_atom(self):
        if self.value == "(":
            self.gettoken()
            a, z = self.parse_rhs()
            self.expect(token.OP, ")")
            return a, z
        elif self.type in (token.NAME, token.STRING):
            a = NFAState()
            z = NFAState()
            a.addarc(z, self.value)
            self.gettoken()
            return a, z
        else:
            self.raise_error("expected (...) or NAME or STRING, got %s/%s",
                             self.type, self.value)

    def expect(self, type, value=None):
        if self.type != type or (value is not None and self.value != value):
            self.raise_error("expected %s/%s, got %s/%s",
                             type, value, self.type, self.value)
        value = self.value
        self.gettoken()
        return value

    def gettoken(self):
        tup = next(self.generator)
        while tup[0] in (token.COMMENT, token.NL):
            tup = next(self.generator)
        self.type, self.value, self.begin, prefix = tup

    def raise_error(self, msg, *args):
        if args:
            try:
                msg = msg % args
            except:
                msg = " ".join([msg] + list(map(str, args)))
        line = open(self.filename).readlines()[self.begin[0]]
        raise SyntaxError(msg, (self.filename, self.begin[0],
                                self.begin[1], line))

class NFAState(object):
    def __init__(self):
        self.arcs = []  # list of (label, NFAState) pairs

    def addarc(self, next, label=None):
        assert label is None or isinstance(label, str)
        assert isinstance(next, NFAState)
        self.arcs.append((label, next))

class DFAState(object):
    def __init__(self, nfaset, final):
        assert isinstance(nfaset, dict)
        assert isinstance(next(iter(nfaset)), NFAState)
        assert isinstance(final, NFAState)
        self.nfaset = nfaset
        self.isfinal = final in nfaset
        self.arcs = {}  # map from label to DFAState

    def addarc(self, next, label):
        assert isinstance(label, str)
        assert label not in self.arcs
        assert isinstance(next, DFAState)
        self.arcs[label] = next

    def unifystate(self, old, new):
        for label, next in self.arcs.items():
            if next is old:
                self.arcs[label] = new

    def __eq__(self, other):
        # Equality test -- ignore the nfaset instance variable
        assert isinstance(other, DFAState)
        if self.isfinal != other.isfinal:
            return False
        # Can't just return self.arcs == other.arcs, because that
        # would invoke this method recursively, with cycles...
        if len(self.arcs) != len(other.arcs):
            return False
        for label, next in self.arcs.items():
            if next is not other.arcs.get(label):
                return False
        return True

    __hash__ = None  # For Py3 compatibility.

def generate_grammar(filename="Grammar.txt"):
    p = ParserGenerator(filename)
    return p.make_grammar()