bpo-24960: use pkgutil.get_data in lib2to3 to read pickled grammar fi… · python/cpython@c1b8eb8 (original) (raw)

File tree

5 files changed

lines changed

5 files changed

lines changed

Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@
20 20 import io
21 21 import os
22 22 import logging
23 +import pkgutil
23 24 import sys
24 25
25 26 # Pgen imports
@@ -143,6 +144,26 @@ def _newer(a, b):
143 144 return os.path.getmtime(a) >= os.path.getmtime(b)
144 145
145 146
147 +def load_packaged_grammar(package, grammar_source):
148 +"""Normally, loads a pickled grammar by doing
149 + pkgutil.get_data(package, pickled_grammar)
150 + where *pickled_grammar* is computed from *grammar_source* by adding the
151 + Python version and using a ``.pickle`` extension.
152 +
153 + However, if *grammar_source* is an extant file, load_grammar(grammar_source)
154 + is called instead. This facilities using a packaged grammar file when needed
155 + but preserves load_grammar's automatic regeneration behavior when possible.
156 +
157 + """
158 +if os.path.isfile(grammar_source):
159 +return load_grammar(grammar_source)
160 +pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
161 +data = pkgutil.get_data(package, pickled_name)
162 +g = grammar.Grammar()
163 +g.loads(data)
164 +return g
165 +
166 +
146 167 def main(*args):
147 168 """Main program, when run as a script: produce grammar pickle files.
148 169
Original file line number Diff line number Diff line change
@@ -108,6 +108,10 @@ def load(self, filename):
108 108 d = pickle.load(f)
109 109 self.__dict__.update(d)
110 110
111 +def loads(self, pkl):
112 +"""Load the grammar tables from a pickle bytes object."""
113 +self.__dict__.update(pickle.loads(pkl))
114 +
111 115 def copy(self):
112 116 """
113 117 Copy the grammar.
Original file line number Diff line number Diff line change
@@ -29,12 +29,12 @@ def __init__(self, grammar):
29 29 setattr(self, name, symbol)
30 30
31 31
32 -python_grammar = driver.load_grammar(_GRAMMAR_FILE)
32 +python_grammar = driver.load_packaged_grammar("lib2to3", _GRAMMAR_FILE)
33 33
34 34 python_symbols = Symbols(python_grammar)
35 35
36 36 python_grammar_no_print_statement = python_grammar.copy()
37 37 del python_grammar_no_print_statement.keywords["print"]
38 38
39 -pattern_grammar = driver.load_grammar(_PATTERN_GRAMMAR_FILE)
39 +pattern_grammar = driver.load_packaged_grammar("lib2to3", _PATTERN_GRAMMAR_FILE)
40 40 pattern_symbols = Symbols(pattern_grammar)
Original file line number Diff line number Diff line change
@@ -12,7 +12,10 @@
12 12 from test.support import verbose
13 13
14 14 # Python imports
15 +import importlib
16 +import operator
15 17 import os
18 +import pickle
16 19 import shutil
17 20 import subprocess
18 21 import sys
@@ -99,6 +102,18 @@ def test_load_grammar_from_subprocess(self):
99 102 finally:
100 103 shutil.rmtree(tmpdir)
101 104
105 +def test_load_packaged_grammar(self):
106 +modname = __name__ + '.load_test'
107 +class MyLoader:
108 +def get_data(self, where):
109 +return pickle.dumps({'elephant': 19})
110 +class MyModule:
111 +__file__ = 'parsertestmodule'
112 +__spec__ = importlib.util.spec_from_loader(modname, MyLoader())
113 +sys.modules[modname] = MyModule()
114 +self.addCleanup(operator.delitem, sys.modules, modname)
115 +g = pgen2_driver.load_packaged_grammar(modname, 'Grammar.txt')
116 +self.assertEqual(g.elephant, 19)
102 117
103 118
104 119 class GrammarTest(support.TestCase):
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
1 +2to3 and lib2to3 can now read pickled grammar files using pkgutil.get_data()
2 +rather than probing the filesystem. This lets 2to3 and lib2to3 work when run
3 +from a zipfile.