        DATA = [
            ["#affected"],
            ["1"],
            [2],
            ["N/A"],
        ]
        EXPECTED_VALUES = [
            ["N/A"],
        ]
        filtered = hxl.data(DATA).count(aggregators='max(#affected)')
        self.assertEqual(EXPECTED_VALUES, filtered.values)

"""
Unit tests for filters
David Megginson
April 2015

"""

import unittest

import datetime, hxl

# Mock URL access so that tests work offline
from . import URL_MOCK_TARGET, URL_MOCK_OBJECT
from unittest.mock import patch

# Base data for tests

DATA = [
    ['Organisation', 'Cluster', 'District', 'Affected'],
    ['#org', '#sector+list', '#adm1', '#affected'],
    ['NGO A', 'WASH', 'Coast', '200'],
    ['NGO B', 'Education', 'Plains', '100'],
    ['NGO B', 'Education', 'Coast', '300'],
    ['NGO A', 'Education, Protection', 'Plains', '150'],
]

class AbstractBaseFilterTest(unittest.TestCase):
    """Base class for all tests."""

    def setUp(self):
        # use a cache filter so that we can run tests multiple times
        super().setUp()
        self.source = hxl.data(DATA).cache()

# Test compiling from a recipe

class TestRecipe(AbstractBaseFilterTest):

    def test_spec(self):
        filtered = hxl.data({
            'input': DATA
        })
        self.assertEqual(filtered.values, DATA[2:])

    def test_recursive(self):
        # try appending a dataset to itself
        input = {
            'input': DATA
        }
        filtered = hxl.data(input).recipe([
            {
                'filter': 'append',
                'append_sources': {
                    'input': DATA,
                    'recipe': [
                        {
                            'filter': 'with_rows',
                            'queries': 'org=ngo b'
                        }
                    ]
                }
            }
        ])
        self.assertEqual(filtered.values, DATA[2:] + DATA[3:5])

    def test_multiple(self):
        filtered = self.source.recipe([
            {
                'filter': 'without_rows',
                'queries': 'adm1=Plains'
            },
            {
                'filter': 'sort',
                'tags': 'sector'
            }
        ])
        self.assertEqual(filtered.values, [DATA[4], DATA[2]])

    def test_json(self):
        # test using a literal JSON string for the recipe
        filtered = self.source.recipe('{"filter": "cache"}')
        self.assertEqual(type(filtered).__name__, 'CacheFilter')

    # test individual filter types in the recipe
    # all of these specify only a single filter as a dict

    def test_add_columns(self):
        filtered = self.source.recipe({
            'filter': 'add_columns',
            'specs': '#foo=bar'
        })
        self.assertEqual(type(filtered).__name__, 'AddColumnsFilter')

    def test_append(self):
        filtered = self.source.recipe({
            'filter': 'append',
            'append_sources': DATA
        })
        self.assertEqual(type(filtered).__name__, 'AppendFilter')

    @patch(URL_MOCK_TARGET, new=URL_MOCK_OBJECT)
    def test_append_external(self):
        filtered = self.source.recipe({
            'filter': 'append_external_list',
            'source_list_url': 'http://example.org/append-source-list.csv'
        })
        self.assertEqual(type(filtered).__name__, 'AppendFilter')

    def test_cache(self):
        filtered = self.source.recipe({'filter': 'cache'})
        self.assertEqual(type(filtered).__name__, 'CacheFilter')

    def test_clean_data(self):
        filtered = self.source.recipe({
            'filter': 'clean_data',
            'number_format': '0.2f'
        })
        self.assertEqual(type(filtered).__name__, 'CleanDataFilter')
        self.assertEqual('0.2f', filtered.number_format)

    def test_count(self):
        filtered = self.source.recipe({
            'filter': 'count',
            'patterns': 'sector',
            'aggregators': 'sum(affected)'
        })
        self.assertEqual(type(filtered).__name__, 'CountFilter')

    def test_dedup(self):
        filtered = self.source.recipe({'filter': 'dedup'})
        self.assertEqual(type(filtered).__name__, 'DeduplicationFilter')

    def test_expand_lists(self):
        filtered = self.source.recipe({'filter': 'expand_lists'})
        self.assertEqual(type(filtered).__name__, 'ExpandListsFilter')

    def test_explode(self):
        filtered = self.source.recipe({'filter': 'explode'})
        self.assertEqual(type(filtered).__name__, 'ExplodeFilter')

    def test_implode(self):
        filtered = self.source.recipe({
            'filter': 'implode',
            'label_pattern': "#date+year",
            'value_pattern': "#affected"
        })
        self.assertEqual(type(filtered).__name__, 'ImplodeFilter')
        self.assertEqual(filtered.label_pattern, "#date+year")
        self.assertEqual(filtered.value_pattern, "#affected")

    def test_fill_data(self):
        filtered = self.source.recipe({'filter': 'fill_data'})
        self.assertEqual(type(filtered).__name__, 'FillDataFilter')

    def test_merge_data(self):
        filtered = self.source.recipe({
            'filter': 'merge_data',
            'merge_source': DATA,
            'keys': 'sector',
            'tags': 'org'
        })
        self.assertEqual(type(filtered).__name__, 'MergeDataFilter')

    def test_rename_columns(self):
        filtered = self.source.recipe({
            'filter': 'rename_columns',
            'specs': '#foo:#bar'
        })
        self.assertEqual(type(filtered).__name__, 'RenameFilter')

    def test_replace_data(self):
        filtered = self.source.recipe({
            'filter': 'replace_data',
            'original': 'foo',
            'replacement': 'bar'
        })
        self.assertEqual(type(filtered).__name__, 'ReplaceDataFilter')

    def test_replace_data_map(self):
        filtered = self.source.recipe({
            'filter': 'replace_data_map',
            'map_source': [
                ['#x_pattern', '#x_substitution', '#x_tag'],
                ['NGO C', 'NGO Charlie', 'org']
            ]
        })
        self.assertEqual(type(filtered).__name__, 'ReplaceDataFilter')

    def test_jsonpath(self):
        filtered = self.source.recipe({'filter': 'jsonpath', 'path': 'a.b'})
        self.assertEqual(type(filtered).__name__, 'JSONPathFilter')

    def test_sort(self):
        filtered = self.source.recipe({'filter': 'sort'})
        self.assertEqual(type(filtered).__name__, 'SortFilter')

    def test_with_columns(self):
        filtered = self.source.recipe({
            'filter': 'with_columns',
            'includes': 'sector'
        })
        self.assertEqual(type(filtered).__name__, 'ColumnFilter')

    def test_with_rows(self):
        filtered = self.source.recipe({
            'filter': 'with_rows',
            'queries': 'sector=WASH'
        })
        self.assertEqual(type(filtered).__name__, 'RowFilter')

    def test_without_columns(self):
        filtered = self.source.recipe({
            'filter': 'without_columns',
            'excludes': 'sector',
            'skip_untagged': True
        })
        self.assertEqual(type(filtered).__name__, 'ColumnFilter')
        self.assertTrue(filtered.skip_untagged)

    def test_without_rows(self):
        filtered = self.source.recipe({
            'filter': 'without_rows',
            'queries': 'sector=WASH'
        })
        self.assertEqual(type(filtered).__name__, 'RowFilter')

# Test classes

class TestAddColumnsFilter(AbstractBaseFilterTest):

    spec = 'Country#country=Country A'

    def test_before(self):
        self.assertEqual(
            ['#country', '#org', '#sector+list', '#adm1', '#affected'],
            self.source.add_columns(self.spec, True).display_tags
        )

    def test_after(self):
        self.assertEqual(
            ['#org', '#sector+list', '#adm1', '#affected', '#country'],
            self.source.add_columns(self.spec).display_tags
        )

    def test_headers(self):
        self.assertEqual(
            DATA[0] + ['Country'],
            self.source.add_columns(self.spec).headers
        )

    def test_rows(self):
        self.assertEqual(
            [values + ['Country A'] for values in DATA[2:]],
            self.source.add_columns(self.spec).values
        )

    def test_dynamic_value(self):
        self.assertEqual(
            [values + [values[1]] for values in DATA[2:]],
            self.source.add_columns('Country#country={{#sector}}').values
        )

    def test_formula(self):
        self.assertEqual(
            [values + [values[3]] for values in DATA[2:]],
            self.source.add_columns('#affected+total={{sum(#affected)}}').values
        )

class TestAppendFilter(AbstractBaseFilterTest):

    APPEND_DATA = [
        ['Org', 'Targeted', 'Sector 1', 'Sector 2'],
        ['#org', '#targeted', '#sector+list', '#sector+list'],
        ['NGO A', '200', 'WASH', ''],
        ['NGO C', '500', 'Health', 'Food']
    ]

    COMBINED_DATA = [
        ['Organisation', 'Cluster', 'District', 'Affected', 'Targeted', 'Sector 2'],
        ['#org', '#sector+list', '#adm1', '#affected', '#targeted', '#sector+list'],
        ['NGO A', 'WASH', 'Coast', '200', '', ''],
        ['NGO B', 'Education', 'Plains', '100', '', ''],
        ['NGO B', 'Education', 'Coast', '300', '', ''],
        ['NGO A', 'Education, Protection', 'Plains', '150', '', ''],
        ['NGO A', 'WASH', '', '', '200', ''],
        ['NGO C', 'Health', '', '', '500', 'Food']
    ]

    COMBINED_DATA_ORIG_COLUMNS = [
        ['Organisation', 'Cluster', 'District', 'Affected'],
        ['#org', '#sector+list', '#adm1', '#affected'],
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', 'Education', 'Coast', '150'],
        ['NGO A', 'WASH', '', ''],
        ['NGO C', 'Health', '', '']
    ]

    COMBINED_DATA_FILTERED = [
        ['Organisation', 'Cluster', 'District', 'Affected', 'Targeted', 'Sector 2'],
        ['#org', '#sector+list', '#adm1', '#affected', '#targeted', '#sector+list'],
        ['NGO A', 'WASH', 'Coast', '200', '', ''],
        ['NGO B', 'Education', 'Plains', '100', '', ''],
        ['NGO B', 'Education', 'Coast', '300', '', ''],
        ['NGO A', 'Education, Protection', 'Plains', '150', '', ''],
        ['NGO C', 'Health', '', '', '500', 'Food']
    ]

    def setUp(self):
        super().setUp()
        self.append_source = hxl.data(TestAppendFilter.APPEND_DATA)

    def test_headers(self):
        self.assertEqual(self.COMBINED_DATA[0], self.source.append(self.append_source).headers)
        self.assertEqual(self.COMBINED_DATA_ORIG_COLUMNS[0], self.source.append(self.append_source, False).headers)

    def test_columns(self):
        self.assertEqual(self.COMBINED_DATA[1], self.source.append(self.append_source).display_tags)

    def test_values(self):
        self.assertEqual(self.COMBINED_DATA[2:], self.source.append(self.append_source).values)

    @patch(URL_MOCK_TARGET, new=URL_MOCK_OBJECT)
    def test_external(self):
        EXPECTED_VALUES = [
            ['NGO A', 'WASH', 'Coast', '200', '', ''],
            ['NGO B', 'Education', 'Plains', '100', '', ''],
            ['NGO B', 'Education', 'Coast', '300', '', ''],
            ['NGO A', 'Education, Protection', 'Plains', '150', '', ''],
            ['NGO A', ' WASH', '', '', ' 200', ' '],
            ['NGO C', ' Health', '', '', ' 500', ' Food'],
            ['NGO A', ' Education', '', '', ' 300', ' '],
            ['NGO C', ' Protection', '', '', ' 100', ' Food']
        ]
        filtered = self.source.append_external_list('http://example.org/append-source-list.csv')
        self.assertEqual(EXPECTED_VALUES, filtered.values)

    def test_queries(self):
        pass # need a new query

class TestCacheFilter(AbstractBaseFilterTest):

    def test_headers(self):
        self.assertEqual(DATA[0], self.source.cache().headers)

    def test_columns(self):
        self.assertEqual(DATA[1], self.source.cache().display_tags)

    def test_rows(self):
        self.assertEqual(DATA[2:], self.source.cache().values)

    def test_repeat(self):
        # Test repeating a cache filter directly
        source = hxl.data(DATA).cache()
        rows1 = [row.values for row in source]
        rows2 = [row.values for row in source]
        self.assertEqual(4, len(rows1))
        self.assertEqual(rows1, rows2)

    def test_repeat_sub(self):
        # Test repeating a cache filter backing another filter
        source = hxl.data(DATA).cache().with_rows('org=NGO A')
        rows1 = [row.values for row in source]
        rows2 = [row.values for row in source]
        self.assertEqual(2, len(rows1))
        self.assertEqual(rows1, rows2)

class TestCleanDataFilter(AbstractBaseFilterTest):

    def test_whitespace(self):
        DATA_IN = [
            ['Organisation', 'Cluster', 'District', 'Count'],
            ['#org', '#sector+list', '#adm1', '#meta+count'],
            ['NGO A', '  WASH', 'Coast', '200'],
            ['NGO B', 'Education  ', 'Plains', '100'],
            ['NGO B', 'Child    Protection', 'Coast', '300']
        ]
        DATA_OUT = [
            ['NGO A', 'WASH', 'Coast', '200'],
            ['NGO B', 'Education', 'Plains', '100'],
            ['NGO B', 'Child Protection', 'Coast', '300']
        ]
        self.assertEqual(DATA_OUT, hxl.data(DATA_IN).clean_data(whitespace='sector').values)

    def test_numbers(self):
        DATA_IN = [
            ['Organisation', 'Cluster', 'District', 'Count'],
            ['#org', '#sector+list', '#adm1', '#meta+count'],
            ['NGO A', 'WASH', 'Coast', '  200'],
            ['NGO B', 'Education', 'Plains', '1,100 '],
            ['NGO B', 'Child Protection', 'Coast', 300],
            ['NGO A', 'Logistics', 'Coast', '1.7E5']
        ]
        DATA_OUT = [
            ['NGO A', 'WASH', 'Coast', '200'],
            ['NGO B', 'Education', 'Plains', '1100'],
            ['NGO B', 'Child Protection', 'Coast', '300'],
            ['NGO A', 'Logistics', 'Coast', '170000']
        ]
        DATA_OUT_FORMATTED = [
            ['NGO A', 'WASH', 'Coast', '200.00'],
            ['NGO B', 'Education', 'Plains', '1100.00'],
            ['NGO B', 'Child Protection', 'Coast', '300.00'],
            ['NGO A', 'Logistics', 'Coast', '170000.00']
        ]
        self.assertEqual(DATA_OUT, hxl.data(DATA_IN).clean_data(number='meta+count').values)
        self.assertEqual(
            DATA_OUT_FORMATTED,
            hxl.data(DATA_IN).clean_data(number='meta+count', number_format='0.2f').values
        )

    def test_dates(self):
        DATA_IN = [
            ['Organisation', 'Cluster', 'District', 'Date'],
            ['#org', '#sector+list', '#adm1', '#date'],
            ['NGO A', 'WASH', 'Coast', 'January 1 2015'],
            ['NGO B', 'Education', 'Plains', '2/2/15'],
            ['NGO B', 'Child Protection', 'Coast', '1 Mar/15']
        ]
        DATA_OUT = [
            ['NGO A', 'WASH', 'Coast', '2015-01-01'],
            ['NGO B', 'Education', 'Plains', '2015-02-02'],
            ['NGO B', 'Child Protection', 'Coast', '2015-03-01']
        ]
        DATA_OUT_WEEK = [
            ['NGO A', 'WASH', 'Coast', '2015-W01'],
            ['NGO B', 'Education', 'Plains', '2015-W06'],
            ['NGO B', 'Child Protection', 'Coast', '2015-W09']
        ]
        DATA_OUT_MONTH = [
            ['NGO A', 'WASH', 'Coast', '2015-01'],
            ['NGO B', 'Education', 'Plains', '2015-02'],
            ['NGO B', 'Child Protection', 'Coast', '2015-03']
        ]
        DATA_OUT_YEAR = [
            ['NGO A', 'WASH', 'Coast', '2015'],
            ['NGO B', 'Education', 'Plains', '2015'],
            ['NGO B', 'Child Protection', 'Coast', '2015']
        ]
        self.assertEqual(DATA_OUT, hxl.data(DATA_IN).clean_data(date='date').values)
        self.assertEqual(DATA_OUT_WEEK, hxl.data(DATA_IN).clean_data(date='date', date_format="%Y-W%V").values)
        self.assertEqual(DATA_OUT_MONTH, hxl.data(DATA_IN).clean_data(date='date', date_format="%Y-%m").values)
        self.assertEqual(DATA_OUT_YEAR, hxl.data(DATA_IN).clean_data(date='date', date_format="%Y").values)

    def test_custom_dates(self):
        # User-supplied data formats
        DATA_IN = [
            ['Organisation', 'Cluster', 'District', 'Date'],
            ['#org', '#sector+list', '#adm1', '#date'],
            ['NGO A', 'WASH', 'Coast', 'January 1 2015'],
            ['NGO B', 'Education', 'Plains', '1/1/15'],
            ['NGO B', 'Child Protection', 'Coast', '1 Jan/15']
        ]
        DATA_OUT_Y = [
            ['NGO A', 'WASH', 'Coast', '2015'],
            ['NGO B', 'Education', 'Plains', '2015'],
            ['NGO B', 'Child Protection', 'Coast', '2015']
        ]
        DATA_OUT_Y_M = [
            ['NGO A', 'WASH', 'Coast', '2015-01'],
            ['NGO B', 'Education', 'Plains', '2015-01'],
            ['NGO B', 'Child Protection', 'Coast', '2015-01']
        ]
        self.assertEqual(DATA_OUT_Y, hxl.data(DATA_IN).clean_data(date='date', date_format='%Y').values)
        self.assertEqual(DATA_OUT_Y_M, hxl.data(DATA_IN).clean_data(date='date', date_format='%Y-%m').values)

    def test_date_ddmm(self):
        DATA_IN = [
            ['#date'],
            ['11-14-15'],
            ['13-11-15'],
            ['11-17-15'],
            ['09-11-15'] # ambiguous
        ]
        EXPECTED = [
            ['#date'],
            ['2015-11-14'],
            ['2015-11-13'],
            ['2015-11-17'],
            ['2015-09-11'] # this is the one that is has to get right
        ]
        source = hxl.data(DATA_IN)
        self.assertEqual(EXPECTED[1:], source.clean_data(date='date').values)

    def test_date_mmdd(self):
        DATA_IN = [
            ['#date'],
            ['14-11-15'],
            ['11-13-15'],
            ['17-11-15'],
            ['09-11-15'] # ambiguous
        ]
        EXPECTED = [
            ['#date'],
            ['2015-11-14'],
            ['2015-11-13'],
            ['2015-11-17'],
            ['2015-11-09'] # this is the one that is has to get right
        ]
        source = hxl.data(DATA_IN)
        self.assertEqual(EXPECTED[1:], source.clean_data(date='date').values)

    def test_date_epoch(self):
        DATA_IN = [
            ['#date'],
            ['1582502400000'],
        ]
        EXPECTED = [
            ['#date'],
            ['2020-02-24'],
        ]
        source = hxl.data(DATA_IN)
        self.assertEqual(EXPECTED[1:], source.clean_data(date='date').values)

    def test_upper_case(self):
        DATA_OUT = [
            ['NGO A', 'WASH', 'Coast', '200'],
            ['NGO B', 'EDUCATION', 'Plains', '100'],
            ['NGO B', 'EDUCATION', 'Coast', '300'],
            ['NGO A', 'EDUCATION, PROTECTION', 'Plains', '150'],
        ]
        self.assertEqual(DATA_OUT, self.source.clean_data(upper='sector').values)

    def test_lower_case(self):
        DATA_OUT = [
            ['NGO A', 'wash', 'Coast', '200'],
            ['NGO B', 'education', 'Plains', '100'],
            ['NGO B', 'education', 'Coast', '300'],
            ['NGO A', 'education, protection', 'Plains', '150']
        ]
        self.assertEqual(DATA_OUT, self.source.clean_data(lower='sector').values)

    def test_latlon(self):
        DATA_IN = [
            ['#foo', '#geo+lat', '#geo+lon', '#geo+coord'],
            ['75W 30 00', '45N 30 00', '75W 30 00', '45N 30 00,75W 30 00'],
        ]
        DATA_OUT = [
            ['75W 30 00', '45.5000', '-75.5000', '45.5000,-75.5000'],
        ]
        self.assertEqual(DATA_OUT, hxl.data(DATA_IN).clean_data(latlon='geo').values)

    def test_purge_malformed_data(self):
        DATA_IN = [
            ['#date', '#affected', '#geo+lat', '#geo+lon'],
            ['1/Mar/2017', 'bad', '45N30', 'bad'],
            ['bad', '2,000', 'bad', '75W30'],
        ]
        DATA_OUT_UNPURGED = [
            ['2017-03-01', 'bad', '45.5000', 'bad'],
            ['bad', '2000', 'bad', '-75.5000'],
        ]
        DATA_OUT_PURGED = [
            ['2017-03-01', '', '45.5000', ''],
            ['', '2000', '', '-75.5000'],
        ]
        self.assertEqual(
            DATA_OUT_UNPURGED,
            hxl.data(DATA_IN).clean_data(date='date', number='affected', latlon='geo', purge=False).values
        )
        self.assertEqual(
            DATA_OUT_PURGED,
            hxl.data(DATA_IN).clean_data(date='date', number='affected', latlon='geo', purge=True).values
        )

    def test_queries(self):
        DATA_OUT = [
            ['NGO A', 'WASH', 'Coast', '200'],
            ['NGO B', 'education', 'Plains', '100'],
            ['NGO B', 'Education', 'Coast', '300'],
            ['NGO A', 'education, protection', 'Plains', '150']
        ]
        self.assertEqual(DATA_OUT, self.source.clean_data(lower='sector', queries='adm1=Plains').values)

class TestColumnFilter(AbstractBaseFilterTest):

    UNTAGGED_DATA_IN = [
        ['Col A', 'Col B', 'Col C'],
        ['#adm1', '', '#org'],
        ['Plains', 'WASH', 'UNICEF'],
        ['Coast', 'Health', 'WHO'],
    ]

    UNTAGGED_DATA_OUT = [
        ['Col A', 'Col C'],
        ['#adm1', '#org'],
        ['Plains', 'UNICEF'],
        ['Coast', 'WHO'],
    ]

    def test_with_columns(self):
        expected = ['#sector+list']
        self.assertEqual(expected, self.source.with_columns('#sector').display_tags)
        self.assertEqual(expected, self.source.with_columns(['#sector']).display_tags)

    def test_without_columns(self):
        expected = ['#org', '#adm1', '#affected']
        self.assertEqual(expected, self.source.without_columns('#sector').display_tags)
        self.assertEqual(expected, self.source.without_columns(['#sector']).display_tags)

    def test_untagged(self):
        source = hxl.data(self.UNTAGGED_DATA_IN).without_columns(skip_untagged=True)
        self.assertEqual(self.UNTAGGED_DATA_OUT[0], source.headers)
        self.assertEqual(self.UNTAGGED_DATA_OUT[1], source.display_tags)
        self.assertEqual(self.UNTAGGED_DATA_OUT[2:], source. values)

class TestCountFilter(AbstractBaseFilterTest):

    def test_count_aggregator(self):
        expected = [
            ['Organisation', 'Total activities'],
            ['#org', '#output+activities'],
            ['NGO A', 2],
            ['NGO B', 2]
        ]
        filtered = self.source.count('org', 'count() as Total activities#output+activities')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_sum_aggregator(self):
        expected = [
            ['Organisation', 'Total activities'],
            ['#org', '#output+activities'],
            ['NGO A', 2],
            ['NGO B', 2]
        ]
        filtered = self.source.count('org', 'count() as Total activities#output+activities')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_average_aggregator(self):
        expected = [
            ['Organisation', 'Average affected'],
            ['#org', '#affected+average'],
            ['NGO A', 175],
            ['NGO B', 200]
        ]
        filtered = self.source.count('org', 'average(#affected) as Average affected#affected+average')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_min_aggregator(self):
        expected = [
            ['Organisation', 'Minimum affected'],
            ['#org', '#affected+min'],
            ['NGO A', '150'],
            ['NGO B', '100']
        ]
        filtered = self.source.count('org', 'min(#affected) as Minimum affected#affected+min')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_min_aggregator_mixed(self):
        """is min aggregator with mixed types"""
        DATA = [
            ["#affected"],
            ["1"],
            [2],
            ["N/A"],
        ]
        EXPECTED_VALUES = [
            ["1"],
        ]
        filtered = hxl.data(DATA).count(aggregators='min(#affected)')
        self.assertEqual(EXPECTED_VALUES, filtered.values)

    def test_max_aggregator(self):
        expected = [
            ['Organisation', 'Maximum affected'],
            ['#org', '#affected+max'],
            ['NGO A', '200'],
            ['NGO B', '300']
        ]
        filtered = self.source.count('org', 'max(#affected) as Maximum affected#affected+max')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_max_aggregator_mixed(self):
        """is max aggregator with mixed types"""

    def test_concat_aggregator(self):
        expected = [
            ['Organisation', 'Districts'],
            ['#org', '#adm1+list'],
            ['NGO A', 'Coast|Plains'],
            ['NGO B', 'Coast|Plains'],
        ]
        filtered = self.source.count('org', 'concat(#adm1) as Districts#adm1+list')
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_aggregator_dates(self):
        DATA_IN = [
            ['#event', '#date'],
            ['Flood', '2017-01-10 00:00:00'],
            ['Flood', '1 Jan 2018'],
            ['Flood', '06/30/2018']
        ]

        # minimum date
        self.assertEqual(
            [['Flood', '2017-01-10 00:00:00']],
            hxl.data(DATA_IN).count('event', 'min(#date)').values
        )

        # maximum date
        self.assertEqual(
            [['Flood', '06/30/2018']],
            hxl.data(DATA_IN).count('event', 'max(#date)').values
        )

    def test_aggregator_strings(self):
        DATA_IN = [
            ['#event', '#sector'],
            ['Flood', 'Food'],
            ['Flood', 'Health'],
            ['Flood', 'Education']
        ]

        # minimum date
        self.assertEqual(
            [['Flood', 'Education']],
            hxl.data(DATA_IN).count('event', 'min(#sector)').values
        )

        # maximum date
        self.assertEqual(
            [['Flood', 'Health']],
            hxl.data(DATA_IN).count('event', 'max(#sector)').values
        )

    def test_whitespace_normalisation(self):
        data = [
            ['#org'],
            ['XXX'],
            ['XXX '],
            ['YYY']
        ]
        expected_values = [
            ['XXX', 2],
            ['YYY', 1]
        ]
        self.assertEqual(expected_values, hxl.data(data).count('#org').values)

    def test_multiple_aggregators(self):
        expected = [
            ['Organisation', 'Minimum affected', 'Maximum affected'],
            ['#org', '#affected+min', '#affected+max'],
            ['NGO A', '150', '200'],
            ['NGO B', '100', '300']
        ]
        filtered = self.source.count('org', [
            'min(#affected) as Minimum affected#affected+min',
            'max(#affected) as Maximum affected#affected+max'
        ])
        self.assertEqual(expected[0], filtered.headers)
        self.assertEqual(expected[1], filtered.display_tags)
        self.assertEqual(expected[2:], filtered.values)

    def test_queries(self):
        expected = [
            ['Education', 1],
            ['WASH', 1]
        ]
        self.assertEqual(expected, self.source.count('#sector', queries='adm1=Coast').values)

class TestDeduplicationFilter (AbstractBaseFilterTest):

    DATA_IN = DATA + DATA[2:] # double up the input data

    DATA_OUT = DATA # should be the same as the original

    DATA_OUT_FILTERED = [
        ['Organisation', 'Cluster', 'District', 'Count'],
        ['#org', '#sector+list', '#adm1', '#meta+count'],
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', 'Education', 'Coast', '300'],
        ['NGO A', 'Education, Protection', 'Plains', '150'],
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO A', 'Education, Protection', 'Plains', '150'],
    ]

    def setUp(self):
        # use a cache filter so that we can run tests multiple times
        super().setUp()
        self.source = hxl.data(self.DATA_IN)

    def test_dedup(self):
        self.assertEqual(self.DATA_OUT[2:], self.source.dedup().values)

    def test_queries(self):
        self.assertEqual(self.DATA_OUT_FILTERED[2:], self.source.dedup(queries='sector=Education').values)

class TestMergeDataFilter(AbstractBaseFilterTest):

    MERGE_IN = [
        ['District', 'P-code'],
        ['#adm1', '#adm1+code'],
        ['coaST', '001'],         # deliberate case variation
        ['   PlÃ¡ins', '002']      # deliberate whitespace variation and accent
    ]

    MERGE_OUT = [
        ['Organisation', 'Cluster', 'District', 'Affected', 'P-code'],
        ['#org', '#sector+list', '#adm1', '#affected', '#adm1+code'],
        ['NGO A', 'WASH', 'Coast', '200', '001'],
        ['NGO B', 'Education', 'Plains', '100', '002'],
        ['NGO B', 'Education', 'Coast', '300', '001'],
        ['NGO A', 'Education, Protection', 'Plains', '150', '002'],
    ]

    MERGE_EXTRA = [
        ['P-code', 'Population'],
        ['#adm1+code', '#population'],
        ['001', '10000'],
        ['002', ''] # deliberately blank
    ]

    MERGE_EXTRA_OUT = [
        ['Organisation', 'Cluster', 'District', 'Affected', 'P-code', 'Population'],
        ['#org', '#sector+list', '#adm1', '#affected', '#adm1+code', '#population'],
        ['NGO A', 'WASH', 'Coast', '200', '001', '10000'],
        ['NGO B', 'Education', 'Plains', '100', '002', ''],
        ['NGO B', 'Education', 'Coast', '300', '001', '10000'],
        ['NGO A', 'Education, Protection', 'Plains', '150', '002', ''],
    ]

    MERGE_DISPLACED_KEY = [
        ['District 1', 'District 2', 'P-code'],
        ['#adm1', '#adm1', '#adm1+code'],
        ['coaST', 'xxx', '001'],         # deliberate case variation
        ['yyy', '   Plains', '002']      # deliberate whitespace variation
    ]

    def setUp(self):
        super().setUp()
        self.merged = self.source.merge_data(hxl.data(self.MERGE_IN), '#adm1-code', '#adm1+code')

    def test_headers(self):
        self.assertEqual(self.MERGE_OUT[0], self.merged.headers)

    def test_tags(self):
        self.assertEqual(self.MERGE_OUT[1], self.merged.display_tags)

    def test_values(self):
        self.assertEqual(self.MERGE_OUT[2:], self.merged.values)

    def test_merge_patterns(self):
        SOURCE_DATA = [
            ['P-code', 'District'],
            ['#adm1+code', '#adm1+name'],
            ['001', 'Coast'],
            ['002', 'Plains'],
        ]
        MERGE_DATA = [
            ['P-code', 'Population (female)', 'Population (male)', 'Population (total)'],
            ['#adm1+code', '#population+f', '#population+m', '#population+total'],
            ['002', '51000', '49000', '100000'],
            ['001', '76000', '74000', '150000'],
        ]
        EXPECTED = [
            ['P-code', 'District', 'Population (female)', 'Population (male)', 'Population (total)'],
            ['#adm1+code', '#adm1+name', '#population+f', '#population+m', '#population+total'],
            ['001', 'Coast', '76000', '74000', '150000'],
            ['002', 'Plains', '51000', '49000', '100000'],
        ]

        result = hxl.data(SOURCE_DATA).merge_data(hxl.data(MERGE_DATA), keys='#adm1+code', tags='#population')
        self.assertEqual(EXPECTED[0], result.headers)
        self.assertEqual(EXPECTED[1], result.display_tags)
        self.assertEqual(EXPECTED[2:], result.values)

        result = hxl.data(SOURCE_DATA).merge_data(hxl.data(MERGE_DATA), keys='#adm1+code', tags='#population+f,#population+m,#population+total')
        self.assertEqual(EXPECTED[0], result.headers)
        self.assertEqual(EXPECTED[1], result.display_tags)
        self.assertEqual(EXPECTED[2:], result.values)

    def test_chaining(self):
        merged_extra = self.merged.merge_data(hxl.data(self.MERGE_EXTRA), '#adm1+code', '#population')
        self.assertEqual(self.MERGE_EXTRA_OUT[2:], merged_extra.values)

    def test_blank_merge(self):
        data1 = hxl.data([
            ['#sector+list', '#org+name', '#org+name'],
            ['Health', '', 'Red Cross']
            ])
        data2 = hxl.data([
            ['#org+name', '#org+code'],
            ['XX', 'YY'],
            ['Red Cross', 'IFRC']
            ])
        expected = [
            ['#sector+list', '#org+name', '#org+name', '#org+code'],
            ['Health', '', 'Red Cross', 'IFRC']
            ]
        merged = data1.merge_data(data2, '#org+name', '#org+code')
        self.assertEqual(expected[1:], merged.values)

    def test_values_displaced_key(self):
        """Test that the filter scans all candidate keys."""
        data1 = hxl.data([
            ['#sector+list', '#org+name', '#org+name'],
            ['Health', 'xxx', 'Red Cross']
            ])
        data2 = hxl.data([
            ['#org+name', '#org+code'],
            ['XX', 'YY'],
            ['Red Cross', 'IFRC']
            ])
        expected = [
            ['#sector+list', '#org+name', '#org+name', '#org+code'],
            ['Health', 'xxx', 'Red Cross', 'IFRC']
            ]
        merged = data1.merge_data(data2, '#org+name', '#org+code')
        self.assertEqual(expected[1:], merged.values)

    def test_queries(self):
        MERGE_IN = [
            ['District', 'P-code', 'Foo'],
            ['#adm1', '#adm1+code', '#foo'],
            ['Coast', '003', 'hack'],
            ['Coast', '001', 'bar'],
            ['Plains', '002', 'hack']
        ]
        MERGE_OUT = [
            ['Organisation', 'Cluster', 'District', 'Count', 'P-code'],
            ['#org', '#sector+list', '#adm1', '#meta+count', '#adm1+code'],
            ['NGO A', 'WASH', 'Coast', '200', '003'],
            ['NGO B', 'Education', 'Plains', '100', '002'],
            ['NGO B', 'Education', 'Coast', '300', '003'],
            ['NGO A', 'Education, Protection', 'Plains', '150', '002'],
        ]
        merged = self.source.merge_data(hxl.data(MERGE_IN), 'adm1-code', 'adm1+code', queries='foo=hack')
        self.assertEqual(MERGE_OUT[2:], merged.values)

class TestRenameFilter(AbstractBaseFilterTest):

    spec = '#sector:Sub-sector#subsector'

    def test_tags(self):
        self.assertEqual(
            ['#org', '#subsector', '#adm1', '#affected'],
            self.source.rename_columns(self.spec).tags
        )

    def test_headers(self):
        self.assertEqual(
            ['Organisation', 'Sub-sector', 'District', 'Affected'],
            self.source.rename_columns(self.spec).headers
        )

    def test_header_selector(self):
        # wide data
        source = hxl.data([
            ["Province", "Women", "Men"],
            ["#adm1", "#affected+label", "#affected+label"],
            ["Coast", 200, 100]
        ])
        expected_tags = ["#adm1", "#affected+f", "#affected+label"]

        self.assertEqual(expected_tags, source.rename_columns('Women#affected+label:Women#affected+f').display_tags)

class TestReplaceFilter(AbstractBaseFilterTest):

    def test_basic_replace(self):
        # should be replaced
        self.assertEqual('Plains District', self.source.replace_data('Plains', 'Plains District', '#adm1').values[1][2])

    def test_empty_replace(self):
        # empty string
        self.assertEqual('', self.source.replace_data('Plains', '', '#adm1').values[1][2])
        self.assertEqual('', self.source.replace_data('^.+$', '', '#adm1', use_regex=True).values[1][2])

        self.assertEqual('', self.source.replace_data('Plains', None, '#adm1').values[1][2])
        self.assertEqual('', self.source.replace_data('^.+$', None, '#adm1', use_regex=True).values[1][2])

    def test_multiple_columns(self):
        # shouldn't throw an except for a list of tag patterns
        self.assertEqual('Plains District', self.source.replace_data('Plains', 'Plains District', '#adm1,#adm2').values[1][2])

    def test_column_ignored(self):
        # shouldn't be replaced
        self.assertEqual('Plains', self.source.replace_data('Plains', 'Plains District', '#org').values[1][2])

    def test_normalised_replace(self):
        # should ignore character case
        self.assertEqual('Plains District', self.source.replace_data('  PLainS   ', 'Plains District', '#adm1').values[1][2])

    def test_all_columns_replace(self):
        # should be replaced (anywhere in row)
        self.assertEqual('Plains District', self.source.replace_data('  PLainS   ', 'Plains District').values[1][2])

    def test_regex_replace(self):
        # not a regex
        self.assertEqual('Plains', self.source.replace_data(r'ains$', 'ains District', '#adm1', use_regex=False).values[1][2])

        # regex
        self.assertEqual('Plains District', self.source.replace_data(r'ains$', 'ains District', '#adm1', True).values[1][2])

        # non-matching regex
        self.assertEqual('Plains', self.source.replace_data(r'^ains', 'ains District', '#adm1', True).values[1][2])

        # substitution
        self.assertEqual('Plains District', self.source.replace_data('(ains)$', r'\1 District', '#adm1', use_regex=True).values[1][2])

    def test_replace_after_append(self):
        # will test with different lengths of value arrays
        NEW_DATA = [
            ['#adm1', '#org'],
            ['Mountains', 'NGO C'],
            ['Plains', 'NGO A']
        ]
        MAPPING = [
            ['#x_pattern', '#x_substitution', '#x_tag'],
            ['NGO C', 'NGO Charlie', 'org']
        ]
        source = self.source.append(NEW_DATA)
        self.assertEqual('NGO Charlie', source.replace_data_map(hxl.data(MAPPING)).values[4][0])

    def test_queries(self):
        result = self.source.replace_data('Coast', 'Coastal District', '#adm1', queries='org=NGO A')
        self.assertEqual('Coastal District', result.values[0][2])
        self.assertEqual('Coast', result.values[2][2])

class TestRowCountFilter(AbstractBaseFilterTest):

    def test_count(self):
        counter = self.source.row_counter()
        for row in counter:
            pass
        self.assertEqual(4, counter.row_count)

    def test_queries(self):
        counter = self.source.row_counter('org=NGO B')
        for row in counter:
            pass
        self.assertEqual(2, counter.row_count)

class TestJSONPathFilter(unittest.TestCase):

    DATA = [
        ['#xxx'],
        ['{"a": {"x": 1, "y": 2}, "b": [1, 2, 3]}'],
        ['bad json']
    ]

    def test_bad_path(self):
        with self.assertRaises(Exception):
            hxl.data(self.DATA).jsonpath("a b c")

    def test_object(self):
        self.assertEqual('1', hxl.data(self.DATA).jsonpath("a.x").values[0][0])

    def test_list(self):
        self.assertEqual('2', hxl.data(self.DATA).jsonpath("b[1]").values[0][0])

    def test_no_match(self):
        self.assertEqual('', hxl.data(self.DATA).jsonpath('no.match').values[0][0])

    def test_filter(self):
        """ Confirm that the JSONPath implementation supports filters. """
        data = [
            ["#x"],
            ['[{"a": 1, "b": 2}, {"a": 3, "b": 4}]'],
        ]
        self.assertEqual("2", hxl.data(data).jsonpath('$[?(@.a=1)].b').values[0][0]);

    def test_bad_json(self):
        # shouldn't raise an exception (but will log a warning)
        self.assertEqual('bad json', hxl.data(self.DATA).jsonpath("a.x").values[1][0])

    def test_outliers(self):
        DATA = [
            ['#xxx'],
            [10]
        ]
        self.assertEqual(10, hxl.data(DATA).jsonpath('a.x').values[0][0])

class TestRowFilter(unittest.TestCase):

    DATA = [
        ['Organisation', 'Cluster', 'District', 'Affected', 'Activity'],
        ['#org', '#sector+list', '#adm1', '#affected', '#activity'],
        ['NGO A', 'WASH', 'Coast', '200', 'Activity 1'],
        ['NGO B', 'Education', 'Plains', '100', 'Activity 2'],
        ['NGO B', 'Education', 'Coast', '300', ''],
        ['NGO A', 'Education, Protection', 'Plains', '150', 'Activity 3']
    ]

    def setUp(self):
        self.source = hxl.data(self.DATA).cache()

    def test_with_rows(self):
        self.assertEqual(self.DATA[3:5], self.source.with_rows(['#sector=education']).values)
        self.assertEqual(self.DATA[3:5], self.source.with_rows('#sector=education').values)

    def test_with_rows_formula(self):
        DATA = [
            ['#foo1', '#foo2'],
            ['100', '50'],
            ['150', '200'],
            ['100', '15'],
        ]
        source = hxl.data(DATA)
        self.assertEqual([DATA[1], DATA[3]], source.with_rows('#foo1 > {{#foo2}}').values)

    def test_with_rows_today(self):
        today = datetime.datetime.utcnow()
        yesterday = today + datetime.timedelta(days=-1)
        tomorrow = today + datetime.timedelta(days=1)
        DATA = [
            ['#date'],
            [yesterday.strftime('%Y-%m-%d')],
            [today.strftime('%Y-%m-%d')],
            [tomorrow.strftime('%Y-%m-%d')],
        ]

        self.assertEqual(
            [DATA[1]], hxl.data(DATA).with_rows('#date < {{today()}}').values
        )

        self.assertEqual(
            [DATA[2]], hxl.data(DATA).with_rows('#date = {{today()}}').values
        )

        self.assertEqual(
            [DATA[3]], hxl.data(DATA).with_rows('#date > {{today()}}').values
        )

    def test_without_rows(self):
        self.assertEqual(self.DATA[3:6], self.source.without_rows(['#sector=wash']).values)
        self.assertEqual(self.DATA[3:6], self.source.without_rows('#sector=wash').values)

    def test_without_rows_formula(self):
        DATA = [
            ['#foo1', '#foo2'],
            ['100', '50'],
            ['150', '200'],
            ['100', '15'],
        ]
        source = hxl.data(DATA)
        self.assertEqual([DATA[2]], source.without_rows('#foo1 > {{#foo2}}').values)

    def test_regex(self):
        self.assertEqual(self.DATA[3:6], self.source.with_rows('#sector~duca').values)
        self.assertEqual(self.DATA[2:3], self.source.with_rows('#sector!~duca').values)

    def test_empty(self):
        self.assertEqual(self.DATA[4:5], self.source.with_rows('#activity is empty').values)

    def test_aggregates(self):
        self.assertEqual(self.DATA[4:5], self.source.with_rows('#affected is max').values)
        self.assertEqual(self.DATA[3:4], self.source.with_rows('#affected is min').values)

    def test_aggregates_mixed_types(self):
        """Test selecting on rows with mixed datatypes"""
        DATA = [
            ["#affected"],
            ["1"],
            [2],
            ["N/A"],
        ]
        self.assertEqual([["1"]], hxl.data(DATA).with_rows('#affected is min').values)
        self.assertEqual([["N/A"]], hxl.data(DATA).with_rows('#affected is max').values)

    def test_masked(self):
        self.assertEqual(self.DATA[2:6], self.source.with_rows('sector=education', mask='org=ngo b').values)

class TestSortFilter(AbstractBaseFilterTest):

    def test_forward(self):
        self.assertEqual(sorted(DATA[2:]), self.source.sort().values)

    def test_backward(self):
        self.assertEqual(sorted(DATA[2:], reverse=True), self.source.sort(reverse=True).values)

    def test_custom_keys(self):
        def key(r):
            return [r[2], r[1]]
        self.assertEqual(sorted(DATA[2:], key=key), self.source.sort(['#adm1', '#sector']).values)

    def test_numeric(self):
        def key(r):
            return float(r[3])
        self.assertEqual(sorted(DATA[2:], key=key), self.source.sort('#affected').values)

    def test_minmax_years(self):
        DATA = [
            ['#date+year', '#affected', '#adm1'],
            ['2016', '200', 'Coast'],
            ['2016', '100', 'Plains'],
            ['2015', '300', 'Coast'],
            ['2015', '200', 'Plains'],
            ['2014', '400', 'Coast'],
            ['2014', '300', 'Plains']
        ]
        self.assertEqual(hxl.data(DATA).with_rows('#date+year is max').values, DATA[1:3])
        self.assertEqual(hxl.data(DATA).with_rows('#date+year is min').values, DATA[5:])

    def test_minmax_numbers(self):
        DATA = [
            ['#date+year', '#affected', '#adm1'],
            ['2016', '200', 'Coast'],
            ['2016', '100', 'Plains'],
            ['2015', '300', 'Coast'],
            ['2015', '200', 'Plains'],
            ['2014', '400', 'Coast'],
            ['2014', '300', 'Plains']
        ]
        self.assertEqual(hxl.data(DATA).with_rows('#affected is max').values, [DATA[5]])
        self.assertEqual(hxl.data(DATA).with_rows('#affected is min').values, [DATA[2]])

    def test_untagged_columns(self):
        DATA = [
            ['#date+year', '#affected', '#adm1', '', ''],
            ['2016', '200', 'Coast', 'xxx', 'yyy'],
            ['2016', '100', 'Plains', 'xxx', 'yyy'],
            ['2015', '300', 'Coast', 'xxx', 'yyy'],
            ['2015', '200', 'Plains', 'xxx', 'yyy', 'xxx', 'yyy'],
            ['2014', '400', 'Coast', 'xxx', 'yyy'],
            ['2014', '300', 'Plains', 'xxx', 'yyy']
        ]

        # should not trigger an exception
        hxl.data(DATA).sort().values

class TestExplodeFilter(AbstractBaseFilterTest):

    # deliberate variation in attribute ordering
    DATA_IN = [
        ['Province', 'Date', 'Girls', 'Boys', 'Women', 'Men'],
        ['#adm1', '#date', '#affected+label+num', '#affected+num+label', '#affected+num+label', '#affected+num+label'],
        ['Coast', '2016', '200', '150', '500', '600'],
        ['Plains', '2016', '300', '450', '800', '750'],
    ]

    DATA_OUT = [
        ['Province', 'Date', 'Group', 'Number affected'],
        ['#adm1', '#date', '#affected+num+header', '#affected+num+value'],
        ['Coast', '2016', 'Girls', '200'],
        ['Coast', '2016', 'Boys', '150'],
        ['Coast', '2016', 'Women', '500'],
        ['Coast', '2016', 'Men', '600'],
        ['Plains', '2016', 'Girls', '300'],
        ['Plains', '2016', 'Boys', '450'],
        ['Plains', '2016', 'Women', '800'],
        ['Plains', '2016', 'Men', '750']
    ]

    def test_headers(self):
        source = hxl.data(self.DATA_IN).explode()
        self.assertEqual(
            ['#adm1', '#date', '#affected+num+header', '#affected+num+value'],
            source.display_tags
        )

    def test_custom_atts(self):
        source = hxl.data(self.DATA_IN).explode('foo', 'bar')
        self.assertEqual(
            ['#adm1', '#date', '#affected+num+foo', '#affected+num+bar'],
            source.display_tags
        )

    def test_content(self):
        self.assertEqual(
            hxl.data(self.DATA_IN).explode().values,
            self.DATA_OUT[2:]
        )

class TestImplodeFilter(AbstractBaseFilterTest):

    DATA_IN = [
        ['Province', 'Date', 'Group', 'Number affected'],
        ['#adm1', '#date', '#group', '#affected'],
        ['Coast', '2016', 'Girls', '200'],
        ['Coast', '2016', 'Boys', '150'],
        ['Coast', '2016', 'Women', '500'],
        ['Coast', '2016', 'Men', '600'],
        ['Plains', '2016', 'Girls', '300'],
        ['Plains', '2016', 'Boys', '450'],
        ['Plains', '2016', 'Women', '800'],
        ['Plains', '2016', 'Men', '750']
    ]

    DATA_OUT = [
        ['Province', 'Date', 'Boys', 'Girls', 'Men', 'Women'],
        ['#adm1', '#date', '#affected+label', '#affected+label', '#affected+label', '#affected+label'],
        ['Coast', '2016', '150', '200', '600', '500'],
        ['Plains', '2016', '450', '300', '750', '800'],
    ]

    def setUp(self):
        self.source = hxl.data(self.DATA_IN).implode(label_pattern="#group", value_pattern="#affected")

    def test_headers (self):
        self.assertEqual(self.DATA_OUT[0], self.source.headers)

    def test_tagspecs (self):
        self.assertEqual(self.DATA_OUT[1], self.source.display_tags)

    def test_values (self):
        self.assertEqual(self.DATA_OUT[2:], self.source.values)

    def test_missing_label(self):
        # non-existant label
        with self.assertRaises(hxl.filters.HXLFilterException):
            source = hxl.data(self.DATA_IN).implode(label_pattern="#foo", value_pattern="#affected")
            source.columns

    def test_missing_value(self):
        # non-existant value
        with self.assertRaises(hxl.filters.HXLFilterException):
            source = hxl.data(self.DATA_IN).implode(label_pattern="#group", value_pattern="#foo")
            source.columns

class TestExpandListsFilter(AbstractBaseFilterTest):
    DATA_IN = [
        ["District", "Organisation", "Cluster"],
        ["#adm1", "#org+name+list", "#sector+cluster+list"],
        ["Coast", "Org A | Org B", "Health | Nutrition | Education"],
        ["Plains", "Org A | Org C", "Shelter"],
    ]

    DATA_OUT = [
        ["District", "Organisation", "Cluster"],
        ["#adm1", "#org+name", "#sector+cluster"],
        ["Coast", "Org A", "Health"],
        ["Coast", "Org A", "Nutrition"],
        ["Coast", "Org A", "Education"],
        ["Coast", "Org B", "Health"],
        ["Coast", "Org B", "Nutrition"],
        ["Coast", "Org B", "Education"],
        ["Plains", "Org A", "Shelter"],
        ["Plains", "Org C", "Shelter"],
    ]

    def setUp(self):
        self.source1 = hxl.data(self.DATA_IN).expand_lists()
        self.source2 = hxl.data(self.DATA_IN).expand_lists(['org', 'sector'])

    def test_headers(self):
        self.assertEqual(self.DATA_OUT[0], self.source1.headers)
        self.assertEqual(self.DATA_OUT[0], self.source2.headers)

    def test_tagspecs(self):
        self.assertEqual(self.DATA_OUT[1], self.source1.display_tags)
        self.assertEqual(self.DATA_OUT[1], self.source2.display_tags)

    def test_values(self):
        self.assertEqual(self.DATA_OUT[2:], self.source1.values)
        self.assertEqual(self.DATA_OUT[2:], self.source2.values)

    def test_separator(self):
        DATA_IN = [
            ["#adm1", "#sector+list"],
            ["Plains", "Health, Education, Protection"],
        ]
        VALUES_OUT = [
            ["Plains", "Health"],
            ["Plains", "Education"],
            ["Plains", "Protection"],
        ]

        source = hxl.data(DATA_IN).expand_lists(separator=",")
        self.assertEqual(VALUES_OUT, source.values)

    def test_correlation(self):
        DATA_IN = [
            ["#adm1+name+list", "#adm1+code+list", "#sector"],
            ["Plains | Coast | Mountains | Capital", "01 | 02 | 03", "Health"],
        ]
        VALUES_OUT = [
            ["Plains", "01", "Health"],
            ["Coast", "02", "Health"],
            ["Mountains", "03", "Health"],
            ["Capital", "", "Health"],
        ]
        source = hxl.data(DATA_IN).expand_lists(correlate=True)
        self.assertEqual(VALUES_OUT, source.values)

class TestFillDataFilter(AbstractBaseFilterTest):

    DATA_IN = [
        ['Organisation', 'Cluster', 'District', 'Affected'],
        ['#org', '#sector+list', '#adm1', '#affected'],
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', '', 'Coast', '300'],
        ['NGO A', '', '', '150'],
    ]

    VALUES_OUT_ALL = [
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', 'Education', 'Coast', '300'],
        ['NGO A', 'Education', 'Coast', '150'],
    ]

    VALUES_OUT_PATTERN = [
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', 'Education', 'Coast', '300'],
        ['NGO A', 'Education', '', '150'],
    ]

    VALUES_OUT_QUERIES = [
        ['NGO A', 'WASH', 'Coast', '200'],
        ['NGO B', 'Education', 'Plains', '100'],
        ['NGO B', 'Education', 'Coast', '300'],
        ['NGO A', '', '', '150'],
    ]

    def test_fill_all(self):
        """Fill everywhere."""
        self.assertEqual(
            hxl.data(self.DATA_IN).fill_data().values,
            self.VALUES_OUT_ALL
        )

    def test_fill_pattern(self):
        """Fill only in selected columns."""
        self.assertEqual(
            hxl.data(self.DATA_IN).fill_data(patterns='#sector').values,
            self.VALUES_OUT_PATTERN
        )

    def test_fill_queries(self):
        """Fill only in selected rows."""
        self.assertEqual(
            hxl.data(self.DATA_IN).fill_data(queries='org=NGO B').values,
            self.VALUES_OUT_QUERIES
        )

class TestChaining(AbstractBaseFilterTest):

    def test_rowfilter_countfilter(self):
        self.assertEqual(
            [['NGO A', 1]],
            self.source.with_rows('#sector=wash').count('#org').values
        )
        self.assertEqual(
            [['NGO A', 1], ['NGO B', 2]],
            self.source.without_rows('#sector=wash').count('#org').values
        )


try :
        dstrings[ i.getAttribute( "name") ] = i.firstChild.data
    except AttributeError :
        pass

from xml.dom import minidom

MANIFEST = "tools/permissions/AndroidManifest.xml"
STRINGS = "tools/permissions/strings.xml"

manifest_document = minidom.parse( MANIFEST )
strings_document = minidom.parse( STRINGS )

dstrings = {}
for i in strings_document.getElementsByTagName( "string" ) :

for i in manifest_document.getElementsByTagName( "permission" ) :
    label_strings = i.getAttribute( "android:label" )[8:]
    description_strings = i.getAttribute( "android:description" )[8:]

    rdesc = "\"\""
    rlabel = "\"\""

    if label_strings == "" or description_strings == "" :
        if label_strings != "" :
            rlabel = dstrings[ label_strings ]
        elif description_strings != "" :
            rdesc = dstrings[ description_strings ]
    else :
        rlabel = dstrings[ label_strings ]
        rdesc = dstrings[ description_strings ]

    name = i.getAttribute("android:name")
    name = name[ name.rfind(".") + 1: ]

    print "\t\t\"%s\"" % name, ": [", "\"%s\"" % i.getAttribute( "android:protectionLevel" ), ",", rlabel, ",", rdesc, "],"


if self._download_status != "": return self._download_status
        if self._canceled: return ""
        return 100*self._i/float(len(self.uris))

# This file is part of dictmaster

import os
import sqlite3

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from dictmaster.util import mkdir_p, CancelableThread, FLAGS
from dictmaster.queue import RawDbQueue

DEFAULT_THREADCNT=6

class FetcherThread(CancelableThread):
    uris = []
    postdata = None
    no = 0

    _i = 0
    _flag = 0
    _queue = None

    def __init__(self, no, uris, queue, flag, postdata=None, sleep=(1.0, 3.0)):
        super(FetcherThread, self).__init__()
        self.uris, self.postdata = uris, postdata
        self.no = no
        self.sleep = sleep
        self._queue = queue
        self._flag = flag | FLAGS["FETCHED"]
        self._canceled = len(self.uris) == 0

    def filter_data(self, data, uri): return data
    def parse_uri(self, uri): return uri

    def fetch_uri(self, rawid, uri):
        data = self.download_retry(self.parse_uri(uri), self.postdata,
                                   sleep=self.sleep)
        if self._canceled: return
        data = self.filter_data(data, uri)
        self._queue.put((rawid, uri, data, self._flag))
        self._i += 1

    def run(self):
        for rawid, uri in self.uris:
            if self._canceled: break
            self.fetch_uri(rawid, uri)

    def progress(self):

class Fetcher(CancelableThread):
    class FetcherThread(FetcherThread): pass

    plugin = None
    postdata = None

    _subthreads = []
    _flag = FLAGS["RAW_FETCHER"]
    _queue = None
    _fetched = 0

    def __init__(self,
        plugin,
        threadcnt=DEFAULT_THREADCNT,
        postdata=None,
        sleep=(1.0, 3.0)
    ):
        super(Fetcher, self).__init__()
        self._subthreads = [None]*threadcnt
        self._queue = RawDbQueue(plugin.output_db)
        self.plugin = plugin
        self.postdata = postdata
        self.sleep = sleep

    def init_subthreads(self, uris):
        self._subthreads = self._subthreads[:len(uris)]
        for i in range(len(self._subthreads)):
            uri_portion = uris[i::len(self._subthreads)]
            self._subthreads[i] = self.FetcherThread(
                no=i,
                uris=uri_portion,
                queue=self._queue,
                flag=self._flag,
                postdata=self.postdata,
                sleep=self.sleep
            )

    def progress(self):
        if None in self._subthreads: return "Initializing threads..."
        if self._canceled: return self._queue.progress()
        prog = "Fetching... "
        sub_p = [s.progress() for s in self._subthreads]
        if all(type(p) == float for p in sub_p):
            percentage = 100*self._fetched
            percentage += (1-self._fetched)*float(sum(sub_p))/len(sub_p)
            prog += "{:.2f}%".format(percentage)
        elif any(type(p) == str and p[:11] == "Downloading" for p in sub_p):
            for p in sub_p:
                if type(p) == str and p[:11] == "Downloading":
                    prog = p
                    break
        else:
            for i,p in enumerate(sub_p):
                prog += "{}:{} ".format(i, p[:13])
        return prog

    def run(self):
        conn = sqlite3.connect(self.plugin.output_db)
        curs = conn.cursor()
        self._fetched = curs.execute('''
            SELECT COUNT(*) FROM raw WHERE flag & :0 == :0
        ''', [FLAGS["FETCHED"] | self._flag]).fetchone()[0]
        uris = curs.execute('''
            SELECT id, uri FROM raw
            WHERE flag & ? == 0
            AND flag & ? > 0
        ''', (FLAGS["FETCHED"], self._flag)).fetchall()
        self._fetched = float(self._fetched) / (len(uris) + self._fetched)
        conn.close()
        if self._canceled or len(uris) == 0: return
        self.init_subthreads(uris)
        self._queue.start()
        [s.start() for s in self._subthreads]
        [s.join() for s in self._subthreads]
        self._queue.cancel()
        self._queue.join()

    def reset(self):
        conn = sqlite3.connect(self.plugin.output_db)
        c = conn.cursor()
        c.execute('''
            UPDATE raw SET flag = flag & ~?
            WHERE flag & ? == 0
        ''', (FLAGS["FETCHED"], FLAGS["URL_FETCHER"]))
        conn.commit()
        conn.close()

    def cancel(self):
        CancelableThread.cancel(self)
        [s.cancel() for s in self._subthreads]
        self._queue.cancel()

class ZipFetcher(Fetcher):
    class FetcherThread(FetcherThread):
        def fetch_uri(self, rawid, uri):
            data = self.download_retry(self.parse_uri(uri), self.postdata)
            if self._canceled: return
            path = self.to_file(data)
            self._queue.put((rawid, uri, path, self._flag))
            self._i += 1

    def __init__(self, plugin, postdata=None):
        super(ZipFetcher, self).__init__(plugin,threadcnt=1,postdata=postdata)
        self._flag = FLAGS["ZIP_FETCHER"]
        def to_file(fthread, data):
            zdirname = os.path.join(self.plugin.output_directory, "zip")
            path = os.path.join(zdirname, "{}_{}".format(fthread.no, fthread._i))
            with open(path, "wb") as f: f.write(data)
            return path
        self.FetcherThread.to_file = to_file


# This file is part of dictmaster

import os
import sqlite3

try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

from dictmaster.util import mkdir_p, CancelableThread, FLAGS
from dictmaster.queue import RawDbQueue

DEFAULT_THREADCNT=6

class FetcherThread(CancelableThread):
    uris = []
    postdata = None
    no = 0

    _i = 0
    _flag = 0
    _queue = None

    def __init__(self, no, uris, queue, flag, postdata=None, sleep=(1.0, 3.0)):
        super(FetcherThread, self).__init__()
        self.uris, self.postdata = uris, postdata
        self.no = no
        self.sleep = sleep
        self._queue = queue
        self._flag = flag | FLAGS["FETCHED"]
        self._canceled = len(self.uris) == 0

    def filter_data(self, data, uri): return data
    def parse_uri(self, uri): return uri

    def fetch_uri(self, rawid, uri):
        data = self.download_retry(self.parse_uri(uri), self.postdata,
                                   sleep=self.sleep)
        if self._canceled: return
        data = self.filter_data(data, uri)
        self._queue.put((rawid, uri, data, self._flag))
        self._i += 1

    def run(self):
        for rawid, uri in self.uris:
            if self._canceled: break
            self.fetch_uri(rawid, uri)

    def progress(self):
        if self._download_status != "": return self._download_status
        if self._canceled: return ""
        return 100*self._i/float(len(self.uris))

class Fetcher(CancelableThread):
    class FetcherThread(FetcherThread): pass

    plugin = None
    postdata = None

    _subthreads = []
    _flag = FLAGS["RAW_FETCHER"]
    _queue = None
    _fetched = 0

    def __init__(self,
        plugin,
        threadcnt=DEFAULT_THREADCNT,
        postdata=None,
        sleep=(1.0, 3.0)
    ):
        super(Fetcher, self).__init__()
        self._subthreads = [None]*threadcnt
        self._queue = RawDbQueue(plugin.output_db)
        self.plugin = plugin
        self.postdata = postdata
        self.sleep = sleep

    def init_subthreads(self, uris):
        self._subthreads

        for i in range(len(self._subthreads)):
            uri_portion = uris[i::len(self._subthreads)]
            self._subthreads[i] = self.FetcherThread(
                no=i,
                uris=uri_portion,
                queue=self._queue,
                flag=self._flag,
                postdata=self.postdata,
                sleep=self.sleep
            )

    def progress(self):
        if None in self._subthreads: return "Initializing threads..."
        if self._canceled: return self._queue.progress()
        prog = "Fetching... "
        sub_p = [s.progress() for s in self._subthreads]
        if all(type(p) == float for p in sub_p):
            percentage = 100*self._fetched
            percentage += (1-self._fetched)*float(sum(sub_p))/len(sub_p)
            prog += "{:.2f}%".format(percentage)
        elif any(type(p) == str and p[:11] == "Downloading" for p in sub_p):
            for p in sub_p:
                if type(p) == str and p[:11] == "Downloading":
                    prog = p
                    break
        else:
            for i,p in enumerate(sub_p):
                prog += "{}:{} ".format(i, p[:13])
        return prog

    def run(self):
        conn = sqlite3.connect(self.plugin.output_db)
        curs = conn.cursor()
        self._fetched = curs.execute('''
            SELECT COUNT(*) FROM raw WHERE flag & :0 == :0
        ''', [FLAGS["FETCHED"] | self._flag]).fetchone()[0]
        uris = curs.execute('''
            SELECT id, uri FROM raw
            WHERE flag & ? == 0
            AND flag & ? > 0
        ''', (FLAGS["FETCHED"], self._flag)).fetchall()
        self._fetched = float(self._fetched) / (len(uris) + self._fetched)
        conn.close()
        if self._canceled or len(uris) == 0: return
        self.init_subthreads(uris)
        self._queue.start()
        [s.start() for s in self._subthreads]
        [s.join() for s in self._subthreads]
        self._queue.cancel()
        self._queue.join()

    def reset(self):
        conn = sqlite3.connect(self.plugin.output_db)
        c = conn.cursor()
        c.execute('''
            UPDATE raw SET flag = flag & ~?
            WHERE flag & ? == 0
        ''', (FLAGS["FETCHED"], FLAGS["URL_FETCHER"]))
        conn.commit()
        conn.close()

    def cancel(self):
        CancelableThread.cancel(self)
        [s.cancel() for s in self._subthreads]
        self._queue.cancel()

class ZipFetcher(Fetcher):
    class FetcherThread(FetcherThread):
        def fetch_uri(self, rawid, uri):
            data = self.download_retry(self.parse_uri(uri), self.postdata)
            if self._canceled: return
            path = self.to_file(data)
            self._queue.put((rawid, uri, path, self._flag))
            self._i += 1

    def __init__(self, plugin, postdata=None):
        super(ZipFetcher, self).__init__(plugin,threadcnt=1,postdata=postdata)
        self._flag = FLAGS["ZIP_FETCHER"]
        def to_file(fthread, data):
            zdirname = os.path.join(self.plugin.output_directory, "zip")
            path = os.path.join(zdirname, "{}_{}".format(fthread.no, fthread._i))
            with open(path, "wb") as f: f.write(data)
            return path
        self.FetcherThread.to_file = to_file


try:
    import urllib2
except ImportError:
    import urllib.request as urllib2

# This file is part of dictmaster

import os
import sqlite3

from dictmaster.util import mkdir_p, CancelableThread, FLAGS
from dictmaster.queue import RawDbQueue

DEFAULT_THREADCNT=6

class FetcherThread(CancelableThread):
    uris = []
    postdata = None
    no = 0

    _i = 0
    _flag = 0
    _queue = None

    def __init__(self, no, uris, queue, flag, postdata=None, sleep=(1.0, 3.0)):
        super(FetcherThread, self).__init__()
        self.uris, self.postdata = uris, postdata
        self.no = no
        self.sleep = sleep
        self._queue = queue
        self._flag = flag | FLAGS["FETCHED"]
        self._canceled = len(self.uris) == 0

    def filter_data(self, data, uri): return data
    def parse_uri(self, uri): return uri

    def fetch_uri(self, rawid, uri):
        data = self.download_retry(self.parse_uri(uri), self.postdata,
                                   sleep=self.sleep)
        if self._canceled: return
        data = self.filter_data(data, uri)
        self._queue.put((rawid, uri, data, self._flag))
        self._i += 1

    def run(self):
        for rawid, uri in self.uris:
            if self._canceled: break
            self.fetch_uri(rawid, uri)

    def progress(self):
        if self._download_status != "": return self._download_status
        if self._canceled: return ""
        return 100*self._i/float(len(self.uris))

class Fetcher(CancelableThread):
    class FetcherThread(FetcherThread): pass

    plugin = None
    postdata = None

    _subthreads = []
    _flag = FLAGS["RAW_FETCHER"]
    _queue = None
    _fetched = 0

    def __init__(self,
        plugin,
        threadcnt=DEFAULT_THREADCNT,
        postdata=None,
        sleep=(1.0, 3.0)
    ):
        super(Fetcher, self).__init__()
        self._subthreads = [None]*threadcnt
        self._queue = RawDbQueue(plugin.output_db)
        self.plugin = plugin
        self.postdata = postdata
        self.sleep = sleep

    def init_subthreads(self, uris):
        self._subthreads = self._subthreads[:len(uris)]
        for i in range(len(self._subthreads)):
            uri_portion = uris[i::len(self._subthreads)]
            self._subthreads[i] = self.FetcherThread(
                no=i,
                uris=uri_portion,
                queue=self._queue,
                flag=self._flag,
                postdata=self.postdata,
                sleep=self.sleep
            )

    def progress(self):
        if None in self._subthreads: return "Initializing threads..."
        if self._canceled: return self._queue.progress()
        prog = "Fetching... "
        sub_p = [s.progress() for s in self._subthreads]
        if all(type(p) == float for p in sub_p):
            percentage = 100*self._fetched
            percentage += (1-self._fetched)*float(sum(sub_p))/len(sub_p)
            prog += "{:.2f}%".format(percentage)
        elif any(type(p) == str and p[:11] == "Downloading" for p in sub_p):
            for p in sub_p:
                if type(p) == str and p[:11] == "Downloading":
                    prog = p
                    break
        else:
            for i,p in enumerate(sub_p):
                prog += "{}:{} ".format(i, p[:13])
        return prog

    def run(self):
        conn = sqlite3.connect(self.plugin.output_db)
        curs = conn.cursor()
        self._fetched = curs.execute('''
            SELECT COUNT(*) FROM raw WHERE flag & :0 == :0
        ''', [FLAGS["FETCHED"] | self._flag]).fetchone()[0]
        uris = curs.execute('''
            SELECT id, uri FROM raw
            WHERE flag & ? == 0
            AND flag & ? > 0
        ''', (FLAGS["FETCHED"], self._flag)).fetchall()
        self._fetched = float(self._fetched) / (len(uris) + self._fetched)
        conn.close()
        if self._canceled or len(uris) == 0: return
        self.init_subthreads(uris)
        self._queue.start()
        [s.start() for s in self._subthreads]
        [s.join() for s in self._subthreads]
        self._queue.cancel()
        self._queue.join()

    def reset(self):
        conn = sqlite3.connect(self.plugin.output_db)
        c = conn.cursor()
        c.execute('''
            UPDATE raw SET flag = flag & ~?
            WHERE flag & ? == 0
        ''', (FLAGS["FETCHED"], FLAGS["URL_FETCHER"]))
        conn.commit()
        conn.close()

    def cancel(self):
        CancelableThread.cancel(self)
        [s.cancel() for s in self._subthreads]
        self._queue.cancel()

class ZipFetcher(Fetcher):
    class FetcherThread(FetcherThread):
        def fetch_uri(self, rawid, uri):
            data = self.download_retry(self.parse_uri(uri), self.postdata)
            if self._canceled: return
            path = self.to_file(data)
            self._queue.put((rawid, uri, path, self._flag))
            self._i += 1

    def __init__(self, plugin, postdata=None):
        super(ZipFetcher, self).__init__(plugin,threadcnt=1,postdata=postdata)
        self._flag = FLAGS["ZIP_FETCHER"]
        def to_file(fthread, data):
            zdirname = os.path.join(self.plugin.output_directory, "zip")
            path = os.path.join(zdirname, "{}_{}".format(fthread.no, fthread._i))
            with open(path, "wb") as f: f.write(data)
            return path
        self.FetcherThread.to_file = to_file


val1, val2 = _icmpPop(frame)
        if val1 < val2:
            branch(frame, self.offset)

"""
@author: HuRuiFeng
@file: Ificmp.py
@time: 2019/9/15 21:06
@desc: if_icmp<cond>指令把栈顶的两个int变量弹出，然后进行比较，满足条件则跳转
"""
from instructions.base.BranchLogic import branch
from instructions.base.Instruction import BranchInstruction
from rtda import Frame

def _icmpPop(frame: Frame):
    stack = frame.operand_stack
    val2 = stack.pop_numeric()
    val1 = stack.pop_numeric()
    return val1, val2

class IF_ICMPEQ(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 == val2:
            branch(frame, self.offset)

class IF_ICMPNE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 != val2:
            branch(frame, self.offset)

class IF_ICMPLT(BranchInstruction):
    def execute(self, frame):

class IF_ICMPLE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 <= val2:
            branch(frame, self.offset)

class IF_ICMPGT(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 > val2:
            branch(frame, self.offset)

class IF_ICMPGE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 >= val2:
            branch(frame, self.offset)


"""
@author: HuRuiFeng
@file: Ificmp.py
@time: 2019/9/15 21:06
@desc: if_icmp<cond>指令把栈顶的两个int变量弹出，然后进行比较，满足条件则跳转
"""
from instructions.base.BranchLogic import branch
from instructions.base.Instruction import BranchInstruction
from rtda import Frame

def _icmpPop(frame: Frame):
    stack = frame.operand_stack
    val2 = stack.pop_numeric()
    val1 = stack.pop_numeric()
    return val1, val2

class IF_ICMPEQ(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 == val2:
            branch(frame, self.offset)

class IF_ICMPNE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 != val2:
            branch(frame, self.offset)

class IF_ICMPLT(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _

        if val1 < val2:
            branch(frame, self.offset)

class IF_ICMPLE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 <= val2:
            branch(frame, self.offset)

class IF_ICMPGT(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 > val2:
            branch(frame, self.offset)

class IF_ICMPGE(BranchInstruction):
    def execute(self, frame):
        val1, val2 = _icmpPop(frame)
        if val1 >= val2:
            branch(frame, self.offset)


build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs()]
        }
        self.write(response)

import http.client
import os
import urllib.parse

import tornado.web
import prometheus_client

from app.master.slave import SlaveRegistry
from app.util import analytics
from app.util import log
from app.util.conf.configuration import Configuration
from app.util.decorators import authenticated
from app.util.exceptions import ItemNotFoundError
from app.util.url_builder import UrlBuilder
from app.web_framework.cluster_application import ClusterApplication
from app.web_framework.cluster_base_handler import ClusterBaseAPIHandler, ClusterBaseHandler
from app.web_framework.route_node import RouteNode

#   Handler classes are not designed to have __init__ overridden.

class ClusterMasterApplication(ClusterApplication):

    def __init__(self, cluster_master):
        """
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        default_params = {
            'cluster_master': cluster_master,
        }
        # The routes are described using a tree structure.  This is a better representation of a path than a flat list
        #  of strings and allows us to inspect children/parents of a node to generate 'child routes'
        api_v1 = [
            RouteNode(r'v1', _APIVersionOneHandler).add_children([
                RouteNode(r'metrics', _MetricsHandler),
                RouteNode(r'version', _VersionHandler),
                RouteNode(r'build', _BuildsHandler, 'builds').add_children([
                    RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                        RouteNode(r'result', _BuildResultRedirectHandler),
                        RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                        RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                        RouteNode(r'subjob', _SubjobsHandler, 'subjobs').add_children([
                            RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                                RouteNode(r'atom', _AtomsHandler, 'atoms').add_children([
                                    RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                        RouteNode(r'console', _AtomConsoleHandler),
                                    ]),
                                ]),
                                RouteNode(r'result', _SubjobResultHandler),
                            ]),
                        ]),
                    ]),
                ]),
                RouteNode(r'queue', _QueueHandler),
                RouteNode(r'slave', _SlavesHandler, 'slaves').add_children([
                    RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                        RouteNode(r'shutdown', _SlaveShutdownHandler, 'shutdown'),
                        RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                    ]),
                    RouteNode(r'shutdown', _SlavesShutdownHandler, 'shutdown'),
                ]),
                RouteNode(r'eventlog', _EventlogHandler)])]

        api_v2 = [
            RouteNode(r'metrics', _MetricsHandler),
            RouteNode(r'version', _VersionHandler),
            RouteNode(r'builds', _V2BuildsHandler).add_children([
                RouteNode(r'(\d+)', _BuildHandler, 'build').add_children([
                    RouteNode(r'result', _BuildResultRedirectHandler),
                    RouteNode(r'artifacts.tar.gz', _BuildTarResultHandler),
                    RouteNode(r'artifacts.zip', _BuildZipResultHandler),
                    RouteNode(r'subjobs', _V2SubjobsHandler,).add_children([
                        RouteNode(r'(\d+)', _SubjobHandler, 'subjob').add_children([
                            RouteNode(r'atoms', _V2AtomsHandler).add_children([
                                RouteNode(r'(\d+)', _AtomHandler, 'atom').add_children([
                                    RouteNode(r'console', _AtomConsoleHandler),
                                ]),
                            ]),
                            RouteNode(r'result', _SubjobResultHandler),
                        ]),
                    ]),
                ]),
            ]),
            RouteNode(r'queue', _QueueHandler),
            RouteNode(r'slaves', _SlavesHandler).add_children([
                RouteNode(r'(\d+)', _SlaveHandler, 'slave').add_children([
                    RouteNode(r'shutdown', _SlaveShutdownHandler),
                    RouteNode(r'heartbeat', _SlavesHeartbeatHandler),
                ]),
                RouteNode(r'shutdown', _SlavesShutdownHandler),
            ]),
            RouteNode(r'eventlog', _EventlogHandler)]

        root = RouteNode(r'/', _RootHandler)
        root.add_children(api_v1, version=1)
        root.add_children(api_v2, version=2)

        handlers = self.get_all_handlers(root, default_params)
        super().__init__(handlers)

class _ClusterMasterBaseAPIHandler(ClusterBaseAPIHandler):
    def initialize(self, route_node=None, cluster_master=None):
        """
        :type route_node: RouteNode
        :type cluster_master: app.master.cluster_master.ClusterMaster
        """
        self._logger = log.get_logger(__name__)
        self._cluster_master = cluster_master
        super().initialize(route_node)

class _RootHandler(_ClusterMasterBaseAPIHandler):
    pass

class _APIVersionOneHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'master': self._cluster_master.api_representation(),
        }
        self.write(response)

class _VersionHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'version': Configuration['version'],
            'api_version': self.api_version,
        }
        self.write(response)

class _MetricsHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        self.write_text(prometheus_client.exposition.generate_latest(prometheus_client.core.REGISTRY))

class _QueueHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        response = {
            'queue': [build.api_representation() for build in self._cluster_master.active_builds()]
        }
        self.write(response)

class _SubjobsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id):

class _V2SubjobsHandler(_SubjobsHandler):
    def get(self, build_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        response = {
            'subjobs': [subjob.api_representation() for subjob in build.get_subjobs(offset, limit)]
        }
        self.write(response)

class _SubjobHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'subjob': subjob.api_representation()
        }
        self.write(response)

class _SubjobResultHandler(_ClusterMasterBaseAPIHandler):
    def post(self, build_id, subjob_id):
        slave_url = self.decoded_body.get('slave')
        slave = SlaveRegistry.singleton().get_slave(slave_url=slave_url)
        file_payload = self.request.files.get('file')
        if not file_payload:
            raise RuntimeError('Result file not provided')

        slave_executor_id = self.decoded_body.get('metric_data', {}).get('executor_id')
        analytics.record_event(analytics.MASTER_RECEIVED_RESULT, executor_id=slave_executor_id, build_id=int(build_id),
                               subjob_id=int(subjob_id), slave_id=slave.id)

        self._cluster_master.handle_result_reported_from_slave(
            slave_url, int(build_id), int(subjob_id), file_payload[0])
        self._write_status()

    def get(self, build_id, subjob_id):
        # TODO: return the subjob's result archive here?
        self.write({'status': 'not implemented'})

class _AtomsHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.atoms()],
        }
        self.write(response)

class _V2AtomsHandler(_AtomsHandler):
    def get(self, build_id, subjob_id):
        offset, limit = self.get_pagination_params()
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        response = {
            'atoms': [atom.api_representation() for atom in subjob.get_atoms(offset, limit)],
        }
        self.write(response)

class _AtomHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        build = self._cluster_master.get_build(int(build_id))
        subjob = build.subjob(int(subjob_id))
        atoms = subjob.atoms
        response = {
            'atom': atoms[int(atom_id)].api_representation(),
        }
        self.write(response)

class _AtomConsoleHandler(_ClusterMasterBaseAPIHandler):
    def get(self, build_id, subjob_id, atom_id):
        """
        :type build_id: int
        :type subjob_id: int
        :type atom_id: int
        """
        max_lines = int(self.get_query_argument('max_lines', 50))
        offset_line = self.get_query_argument('offset_line', None)

        if offset_line is not None:
            offset_line = int(offset_line)

        try:
            response = self._cluster_master.get_console_output(
                build_id,
                subjob_id,
                atom_id,
                Configuration['results_directory'],
                max_lines,
                offset_line
            )
            self.write(response)
            return
        except ItemNotFoundError as e:
            # If the master doesn't have the atom's console output, it's possible it's currently being worked on,
            # in which case the slave that is working on it may be able to provide the in-progress console output.
            build = self._cluster_master.get_build(int(build_id))
            subjob = build.subjob(int(subjob_id))
            slave = subjob.slave

            if slave is None:
                raise e

            api_url_builder = UrlBuilder(slave.url)
            slave_console_url = api_url_builder.url('build', build_id, 'subjob', subjob_id, 'atom', atom_id, 'console')
            query = {'max_lines': max_lines}

            if offset_line is not None:
                query['offset_line'] = offset_line

            query_string = urllib.parse.urlencode(query)
            self.redirect('{}?{}'.format(slave_console_url, query_string))

class _BuildsHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        build_params = self.decoded_body
        success, response = self._cluster_master.handle_request_for_new_build(build_params)
        status_code = http.client.ACCEPTED if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self):
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds()]
        }
        self.write(response)

class _V2BuildsHandler(_BuildsHandler):
    def get(self):
        offset, limit = self.get_pagination_params()
        response = {
            'builds': [build.api_representation() for build in self._cluster_master.get_builds(offset, limit)]
        }
        self.write(response)

class _BuildHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def put(self, build_id):
        update_params = self.decoded_body
        success, response = self._cluster_master.handle_request_to_update_build(build_id, update_params)
        status_code = http.client.OK if success else http.client.BAD_REQUEST
        self._write_status(response, success, status_code=status_code)

    def get(self, build_id):
        response = {
            'build': self._cluster_master.get_build(int(build_id)).api_representation(),
        }
        self.write(response)

class _BuildResultRedirectHandler(_ClusterMasterBaseAPIHandler):
    """
    Redirect to the actual build results file download URL.
    """
    def get(self, build_id):
        self.redirect('/v1/build/{}/artifacts.tar.gz'.format(build_id))

class _BuildResultHandler(ClusterBaseHandler, tornado.web.StaticFileHandler):
    """
    Download an artifact for the specified build. Note this class inherits from ClusterBaseHandler and
    StaticFileHandler, so the semantics of this handler are a bit different than the other handlers in this file that
    inherit from _ClusterMasterBaseHandler.

    From the Tornado docs: "for heavy traffic it will be more efficient to use a dedicated static file server".
    """
    def initialize(self, route_node=None, cluster_master=None):
        """
        :param route_node: This is not used, it is only a param so we can pass route_node to all handlers without error.
        In other routes, route_node is used to find child routes but filehandler routes will never show child routes.
        :type route_node: RouteNode | None
        :type cluster_master: app.master.cluster_master.ClusterMaster | None
        """
        self._cluster_master = cluster_master
        super().initialize(path=None)  # we will not set the root path until the get() method is called

    def get(self, build_id):
        artifact_file_path = self.get_result_file_download_path(int(build_id))
        self.root, artifact_filename = os.path.split(artifact_file_path)
        self.set_header('Content-Type', 'application/octet-stream')  # this should be downloaded as a binary file
        return super().get(path=artifact_filename)

    def get_result_file_download_path(self, build_id: int):
        raise NotImplementedError

class _BuildTarResultHandler(_BuildResultHandler):
    """Handler for the tar archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.tar.gz for the specified build."""
        return self._cluster_master.get_path_for_build_results_archive(build_id, is_tar_request=True)

class _BuildZipResultHandler(_BuildResultHandler):
    """Handler for the zip archive file"""
    def get_result_file_download_path(self, build_id: int):
        """Get the file path to the artifacts.zip for the specified build."""
        return self._cluster_master.get_path_for_build_results_archive(build_id)

class _SlavesHandler(_ClusterMasterBaseAPIHandler):
    def post(self):
        slave_url = self.decoded_body.get('slave')
        num_executors = int(self.decoded_body.get('num_executors'))
        session_id = self.decoded_body.get('session_id')
        response = self._cluster_master.connect_slave(slave_url, num_executors, session_id)
        self._write_status(response, status_code=201)

    def get(self):

        response = {
            'slaves': [slave.api_representation() for slave in SlaveRegistry.singleton().get_all_slaves_by_id().values()]
        }
        self.write(response)

class _SlaveHandler(_ClusterMasterBaseAPIHandler):
    def get(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        response = {
            'slave': slave.api_representation()
        }
        self.write(response)

    @authenticated
    def put(self, slave_id):
        new_slave_state = self.decoded_body.get('slave', {}).get('state')
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))
        self._cluster_master.handle_slave_state_update(slave, new_slave_state)
        self._cluster_master.update_slave_last_heartbeat_time(slave)

        self._write_status({
            'slave': slave.api_representation()
        })

class _EventlogHandler(_ClusterMasterBaseAPIHandler):
    def get(self):
        # all arguments are optional, so default to None
        since_timestamp = self.get_query_argument('since_timestamp', None)
        since_id = self.get_query_argument('since_id', None)
        self.write({
            'events': analytics.get_events(since_timestamp, since_id),
        })

class _SlaveShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slaves_to_shutdown = [int(slave_id)]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesShutdownHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self):
        shutdown_all = self.decoded_body.get('shutdown_all')
        if shutdown_all:
            slaves_to_shutdown = SlaveRegistry.singleton().get_all_slaves_by_id().keys()
        else:
            slaves_to_shutdown = [int(slave_id) for slave_id in self.decoded_body.get('slaves')]

        self._cluster_master.set_shutdown_mode_on_slaves(slaves_to_shutdown)

class _SlavesHeartbeatHandler(_ClusterMasterBaseAPIHandler):
    @authenticated
    def post(self, slave_id):
        slave = SlaveRegistry.singleton().get_slave(slave_id=int(slave_id))

        # If the slave has been marked dead, but still sends heartbeat, the master does not update the last
        # heartbeat time and the method returns false. Additionally, master responds to the slave with slave
        # status. The slave will treat a is_alive=false response as a heartbeat failure, and die.
        # The reason master returns the status to the slave instead of simply marking the slave as alive is
        # because the master or slave do not maintain an explicit state about when and why the slave was marked
        # dead. It is a lot cleaner for the heartbeat functionality to indicate an heartbeat failure and let the
        # slave make a decision based on that.

        is_alive = self._cluster_master.update_slave_last_heartbeat_time(slave)
        self.write({'is_alive': is_alive})