"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):

		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """

		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


, f2 = self.get_frame(f1_s), self.get_frame(f2_s)

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1

		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:
						new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


"""
@author: <example@nnthink.com>

This module primarily defines the FrameNet class and other associated classes/methods.

"""

from src.frames import *

class TypeSystemException(Exception):
	def __init__(self, message):
		self.message = message

class FrameTypeSystem(object):
	""" Simple type system hierarchy object, with associated methods. Used to navigate inheritance relations in FrameNet. """
	def __init__(self, fn):
		self.roots = []
		self.memoized_roots = dict()

	def build(self, fn):
		for frame in fn.frames:
			print(frame.name)
			root_frames = self.get_root(frame, fn)
			for root_frame in root_frames:
				if not root_frame in self.roots:
					self.roots.append(root_frame)

	def get_roots(self, frame, fn, roots=[]):
		if frame.name in self.memoized_roots:
			return self.memoized_roots[frame.name]
		if len(frame.parents) > 0:
			for parent in frame.parents:
				new = fn.get_frame(parent)
				roots += self.get_root(new, fn, roots)
			self.memoized_roots[frame.name] = list(set(roots))
			return list(set(roots))
		else:
			if frame not in roots:
				self.memoized_roots[frame.name] = frame
				roots.append(frame)
			return roots

	def get_root(self, frame, fn):
		if len(frame.parents) > 0:
			new = fn.get_frame(frame.parents[0])
			return self.get_root(new, fn)
		else:
			return frame

	def get_siblings(self, frame, fn):
		""" Returns siblings of a frame, e.g. the other frames that inherit from the same parent(s). """
		parents = frame.parents
		siblings = []
		for i in parents:
			parent = fn.get_frame(i)
			siblings += parent.children
		return siblings

	def lowest_common_supertype(self, f1, f2, fn):
		""" Returns the common supertype of F1 and F2. Could be used for metaphor, etc. """
		if f1 == f2:
			return f1
		elif fn.subtype(f1, f2):
			return f2
		elif fn.subtype(f2, f1):
			return f1
		elif fn.get_root(f1) != fn.get_root(f2):
			raise TypeSystemException("No shared supertype found...different roots.")
		else:
			f1_parents = f1.parents
			f2_parents = f2.parents
			for i in f1_parents:
				frame = fn.get_frame(i)
				if i in f2_parents:
					return i
				return self.lowest_common_supertype(frame, f2, fn)
			for i in f2_parents:
				frame = fn.get_frame(i)
				if i in f1_parents:
					return i
				return self.lowest_common_supertype(frame, f1, fn)

class FrameNet(object):
	""" Contains a list of frames, as well as mappings from lexical units to associated frames. """
	def __init__(self):
		self.name_to_frames = {}
		self.hierarchy = {}
		self.frames = []
		self.lexemes_to_frames = {}
		self.typesystem = None
		self.definitions_to_frames = {}
		self.xml_definitions_to_frames = {}
		self.ID_to_frames = {}
		self.untagged_lus = {}

	def common_supertype(self, f1, f2):
		""" Returns the lowest common supertype between two frames. Could be useful for metaphor. """
		try:
			return self.typesystem.lowest_common_supertype(f1, f2, self)
		except TypeSystemException as e:
			print(e.message)
			return None

	def subtype_s(self, f1_s, f2_s):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		f1, f2 = self.get_frame(f1_s), self.get_frame(f2_s)
		return self.subtype(f1, f2)

	def subtype(self, f1, f2):
		""" Returns TRUE if f1 inherits from f2 or from one of f2's children. """
		self.get_root(f1) != self.get_root(f2)
		if f1.name in f2.children:
			return True
		else:
			for child in f2.children:
				child_frame = self.get_frame(child)
				if self.subtype(f1, child_frame):
					return True
			return False

	def get_frames_from_lu_set(self, lu_set):
		"""Takes in a set of lus and returns a dictionary of each lu mapped onto frames."""
		final = dict()
		for lu in lu_set:
			final[lu] = self.get_frames_from_lu(lu)
		return final

	def get_frames_from_lu(self, lu):
		""" Returns from from lu String, e.g. "move.v". """
		if lu in self.lexemes_to_frames:
			return self.lexemes_to_frames[lu]

	def get_root(self, frame):
		""" Returns root frame, e.g. "Event". """
		return self.typesystem.get_root(frame, self)

	def get_siblings(self, frame):
		""" Returns frames with the same parent. """
		return self.typesystem.get_siblings(frame, self)

	def get_frame(self, name):
		""" Returns frame from name, e.g. "Motion". """
		if name in self.name_to_frames:
			return self.name_to_frames[name]

	def get_frame_from_id(self, ID):
		""" returns frame from ID. """
		return self.ID_to_frames[ID]

	def add_frame(self, frame):
		""" Adds frame to list, as well as populating lexemes_to_frames with mappings from each lu to that frame. """
		self.name_to_frames[frame.name] = frame
		self.frames.append(frame)
		for lu in frame.lexicalUnits:
			#TODO: How to represent?
			untag = lu.name.split(".")[0]
			if untag not in self.untagged_lus:
				self.untagged_lus[untag] = []
			lex = lu.name
			if lex not in self.lexemes_to_frames:
				self.lexemes_to_frames[lex] = []
			self.lexemes_to_frames[lex].append(frame)
			self.untagged_lus[untag].append(frame)
			self.definitions_to_frames[frame.definition] = frame
			self.xml_definitions_to_frames[frame.xml_definition] = frame
			self.ID_to_frames[int(frame.ID)] = frame

	def frame_from_raw_lemma(self, lemma):
		if lemma in self.untagged_lus:
			return self.untagged_lus[lemma]

	def add_element(self, f1, f2):
		""" Adds element from Frame1 to Frame2. """
		for element in f1.elements:
			if element not in f2.elements:
				f2.elements.append(element)

	def build_relations(self):
		""" For all relations and related_frames, replaces Strings with actual frame objects. """
		for frame in self.frames:
			for relation in frame.relations:
				new_frames = []
				for fr in relation.related_frames:
					related = self.get_frame(fr)
					if related:

		new_frames.append(related)
						if relation.relation_type == "Inherits from":
							self.add_element(related, frame)
				relation.related_frames = new_frames

			new_elements = []
			for element in frame.elements:
				new_element = self.get_frame(element.name)
				element.frame = new_element

	def build_typesystem(self):
		""" Constructs typesystem. """
		self.typesystem = FrameTypeSystem(self)


import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():

    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()


import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cl

return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


if id_ in reviews:
                    continue

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue

                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()

    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue

import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):

                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()


import argparse
import codecs
import csv
import fnmatch
import html
import os
import re

def get_review_filesnames(input_dir):
    for root, dirnames, filenames in os.walk(input_dir):
        for filename in fnmatch.filter(filenames, '*.html

'):
            yield os.path.join(root, filename)

def cleanhtml(htmltext):
    cleanre = re.compile('<.*?>')
    cleantext = re.sub(cleanre, ' ', htmltext)
    return cleantext

summaryre = re.compile(r'innerBubble(.*?)ReportIAP', re.M | re.S)
# old format
# overallratingre = re.compile(r'class="sprite-rating_s_fill rating_s_fill s([0-9])0\"')
overallratingre = re.compile(r'reviewItemInline.*?bubble_([0-9])', re.M | re.S)
reviewtextre = re.compile(r'
(.*?)
', re.M | re.S)
aspectre = re.compile(r'recommend-answer(.*?)
', re.M | re.S)
# old format
# aspectratingre = re.compile(r'sprite-rating_ss_fill rating_ss_fill ss([0-9])0')
aspectratingre = re.compile(r'bubble_([0-9])')
# old format
aspectnamere = re.compile(r'recommend-description">(.*)
(.*)?
')
althotelnamere = re.compile(r'"description" content="(.*)?:')
hotelnamere = re.compile(r'title: "(.*)?"')
idre = re.compile(r'id="rn([0-9]+)"')

def get_aspect_ratings(block):
    rates = list()
    for aspectrating in aspectre.findall(block):
        rating = aspectratingre.findall(aspectrating)[0]
        name = aspectnamere.findall(aspectrating)[0].strip()
        rates.append(':'.join([name, rating]))
    return u';'.join(rates)

def main():
    parser = argparse.ArgumentParser(
        description='TripAdvisor Hotel parser')
    parser.add_argument('-d', '--dir', help='Directory with the data for parsing', required=True)
    parser.add_argument('-o', '--outfile', help='Output file path for saving the reviews in csv format', required=True)

    args = parser.parse_args()

    reviews = set()

    with codecs.open(args.outfile, 'w', encoding='utf8') as out:
        writer = csv.writer(out, lineterminator='\n')
        for filepath in get_review_filesnames(args.dir):
            with codecs.open(filepath, mode='r', encoding='utf8') as file:
                htmlpage = file.read()
            print(filepath)
            try:
                hotelname = hotelnamere.findall(htmlpage)[0].strip()
            except IndexError:
                try:
                    hotelname = oldhotelnamere.findall(htmlpage)[0].strip()
                except IndexError:
                    hotelname = althotelnamere.findall(htmlpage)[0].strip()
            for block in summaryre.findall(htmlpage):
                try:
                    id_ = idre.findall(block)[0]
                except IndexError:
                    continue
                if id_ in reviews:
                    continue
                reviews.add(id_)
                reviewtext = reviewtextre.findall(block)[0]
                overallrating = int(overallratingre.findall(block)[0])
                if overallrating >= 4:
                    binaryrating = 'positive'
                else:
                    binaryrating = 'negative'
                reviewtext = cleanhtml(reviewtext).strip()
                try:
                    reviewtext = html.unescape(reviewtext)
                except Exception:
                    pass
                review = [id_, filepath, hotelname, reviewtext, overallrating, binaryrating]
                writer.writerow(review)

if __name__ == '__main__':
    main()