Machines.py 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255
  1. #=======================================================================
  2. #
  3. # Python Lexical Analyser
  4. #
  5. # Classes for building NFAs and DFAs
  6. #
  7. #=======================================================================
  8. from __future__ import absolute_import
  9. import sys
  10. from .Transitions import TransitionMap
  11. try:
  12. from sys import maxsize as maxint
  13. except ImportError:
  14. from sys import maxint
  15. try:
  16. unichr
  17. except NameError:
  18. unichr = chr
  19. LOWEST_PRIORITY = -maxint
  20. class Machine(object):
  21. """A collection of Nodes representing an NFA or DFA."""
  22. states = None # [Node]
  23. next_state_number = 1
  24. initial_states = None # {(name, bol): Node}
  25. def __init__(self):
  26. self.states = []
  27. self.initial_states = {}
  28. def __del__(self):
  29. #print "Destroying", self ###
  30. for state in self.states:
  31. state.destroy()
  32. def new_state(self):
  33. """Add a new state to the machine and return it."""
  34. s = Node()
  35. n = self.next_state_number
  36. self.next_state_number = n + 1
  37. s.number = n
  38. self.states.append(s)
  39. return s
  40. def new_initial_state(self, name):
  41. state = self.new_state()
  42. self.make_initial_state(name, state)
  43. return state
  44. def make_initial_state(self, name, state):
  45. self.initial_states[name] = state
  46. def get_initial_state(self, name):
  47. return self.initial_states[name]
  48. def dump(self, file):
  49. file.write("Plex.Machine:\n")
  50. if self.initial_states is not None:
  51. file.write(" Initial states:\n")
  52. for (name, state) in sorted(self.initial_states.items()):
  53. file.write(" '%s': %d\n" % (name, state.number))
  54. for s in self.states:
  55. s.dump(file)
  56. class Node(object):
  57. """A state of an NFA or DFA."""
  58. transitions = None # TransitionMap
  59. action = None # Action
  60. action_priority = None # integer
  61. number = 0 # for debug output
  62. epsilon_closure = None # used by nfa_to_dfa()
  63. def __init__(self):
  64. # Preinitialise the list of empty transitions, because
  65. # the nfa-to-dfa algorithm needs it
  66. #self.transitions = {'':[]}
  67. self.transitions = TransitionMap()
  68. self.action_priority = LOWEST_PRIORITY
  69. def destroy(self):
  70. #print "Destroying", self ###
  71. self.transitions = None
  72. self.action = None
  73. self.epsilon_closure = None
  74. def add_transition(self, event, new_state):
  75. self.transitions.add(event, new_state)
  76. def link_to(self, state):
  77. """Add an epsilon-move from this state to another state."""
  78. self.add_transition('', state)
  79. def set_action(self, action, priority):
  80. """Make this an accepting state with the given action. If
  81. there is already an action, choose the action with highest
  82. priority."""
  83. if priority > self.action_priority:
  84. self.action = action
  85. self.action_priority = priority
  86. def get_action(self):
  87. return self.action
  88. def get_action_priority(self):
  89. return self.action_priority
  90. def is_accepting(self):
  91. return self.action is not None
  92. def __str__(self):
  93. return "State %d" % self.number
  94. def dump(self, file):
  95. # Header
  96. file.write(" State %d:\n" % self.number)
  97. # Transitions
  98. # self.dump_transitions(file)
  99. self.transitions.dump(file)
  100. # Action
  101. action = self.action
  102. priority = self.action_priority
  103. if action is not None:
  104. file.write(" %s [priority %d]\n" % (action, priority))
  105. def __lt__(self, other):
  106. return self.number < other.number
  107. class FastMachine(object):
  108. """
  109. FastMachine is a deterministic machine represented in a way that
  110. allows fast scanning.
  111. """
  112. initial_states = None # {state_name:state}
  113. states = None # [state] where state = {event:state, 'else':state, 'action':Action}
  114. next_number = 1 # for debugging
  115. new_state_template = {
  116. '': None, 'bol': None, 'eol': None, 'eof': None, 'else': None
  117. }
  118. def __init__(self):
  119. self.initial_states = {}
  120. self.states = []
  121. def __del__(self):
  122. for state in self.states:
  123. state.clear()
  124. def new_state(self, action=None):
  125. number = self.next_number
  126. self.next_number = number + 1
  127. result = self.new_state_template.copy()
  128. result['number'] = number
  129. result['action'] = action
  130. self.states.append(result)
  131. return result
  132. def make_initial_state(self, name, state):
  133. self.initial_states[name] = state
  134. def add_transitions(self, state, event, new_state, maxint=maxint):
  135. if type(event) is tuple:
  136. code0, code1 = event
  137. if code0 == -maxint:
  138. state['else'] = new_state
  139. elif code1 != maxint:
  140. while code0 < code1:
  141. state[unichr(code0)] = new_state
  142. code0 += 1
  143. else:
  144. state[event] = new_state
  145. def get_initial_state(self, name):
  146. return self.initial_states[name]
  147. def dump(self, file):
  148. file.write("Plex.FastMachine:\n")
  149. file.write(" Initial states:\n")
  150. for name, state in sorted(self.initial_states.items()):
  151. file.write(" %s: %s\n" % (repr(name), state['number']))
  152. for state in self.states:
  153. self.dump_state(state, file)
  154. def dump_state(self, state, file):
  155. # Header
  156. file.write(" State %d:\n" % state['number'])
  157. # Transitions
  158. self.dump_transitions(state, file)
  159. # Action
  160. action = state['action']
  161. if action is not None:
  162. file.write(" %s\n" % action)
  163. def dump_transitions(self, state, file):
  164. chars_leading_to_state = {}
  165. special_to_state = {}
  166. for (c, s) in state.items():
  167. if len(c) == 1:
  168. chars = chars_leading_to_state.get(id(s), None)
  169. if chars is None:
  170. chars = []
  171. chars_leading_to_state[id(s)] = chars
  172. chars.append(c)
  173. elif len(c) <= 4:
  174. special_to_state[c] = s
  175. ranges_to_state = {}
  176. for state in self.states:
  177. char_list = chars_leading_to_state.get(id(state), None)
  178. if char_list:
  179. ranges = self.chars_to_ranges(char_list)
  180. ranges_to_state[ranges] = state
  181. ranges_list = ranges_to_state.keys()
  182. ranges_list.sort()
  183. for ranges in ranges_list:
  184. key = self.ranges_to_string(ranges)
  185. state = ranges_to_state[ranges]
  186. file.write(" %s --> State %d\n" % (key, state['number']))
  187. for key in ('bol', 'eol', 'eof', 'else'):
  188. state = special_to_state.get(key, None)
  189. if state:
  190. file.write(" %s --> State %d\n" % (key, state['number']))
  191. def chars_to_ranges(self, char_list):
  192. char_list.sort()
  193. i = 0
  194. n = len(char_list)
  195. result = []
  196. while i < n:
  197. c1 = ord(char_list[i])
  198. c2 = c1
  199. i += 1
  200. while i < n and ord(char_list[i]) == c2 + 1:
  201. i += 1
  202. c2 += 1
  203. result.append((chr(c1), chr(c2)))
  204. return tuple(result)
  205. def ranges_to_string(self, range_list):
  206. return ','.join(map(self.range_to_string, range_list))
  207. def range_to_string(self, range_tuple):
  208. (c1, c2) = range_tuple
  209. if c1 == c2:
  210. return repr(c1)
  211. else:
  212. return "%s..%s" % (repr(c1), repr(c2))