Fix lots of missing encoding="utf-8" parameters

doc/conf.py

@@ -236,7 +236,7 @@ def pg_context(app, pagename, templatename, ctx, event_arg):
         ctx['lexers'] = sorted(pygments.lexers.get_all_lexers(plugins=False), key=lambda x: x[0].lower())
 
     if pagename in ('styles', 'demo'):
-        with open('examples/example.py') as f:
+        with open('examples/example.py', encoding='utf-8') as f:
             html = f.read()
         lexer = pygments.lexers.get_lexer_for_filename('example.py')
         min_contrasts = test_contrasts.min_contrasts()
@@ -273,7 +273,7 @@ def pg_context(app, pagename, templatename, ctx, event_arg):
 def source_read(app, docname, source):
     # linkify issue / PR numbers in changelog
     if docname == 'docs/changelog':
-        with open('../CHANGES') as f:
+        with open('../CHANGES', encoding='utf-8') as f:
             changelog = f.read()
 
         idx = changelog.find('\nVersion 2.4.2\n')

pygments/formatters/html.py

@@ -635,7 +635,7 @@ def _wrap_full(self, inner, outfile):
             # write CSS file only if noclobber_cssfile isn't given as an option.
             try:
                 if not os.path.exists(cssfilename) or not self.noclobber_cssfile:
-                    with open(cssfilename, "w") as cf:
+                    with open(cssfilename, "w", encoding="utf-8") as cf:
                         cf.write(CSSFILE_TEMPLATE %
                                  {'styledefs': self.get_style_defs('body')})
             except OSError as err:
@@ -721,7 +721,7 @@ def _wrap_tablelinenos(self, inner):
         yield 0, dummyoutfile.getvalue()
         yield 0, '</div>'
         yield 0, '</td></tr></table>'
-        
+
 
     def _wrap_inlinelinenos(self, inner):
         # need a list of lines since we need the width of a single number :(
@@ -946,9 +946,9 @@ def wrap(self, source):
         output = source
         if self.wrapcode:
             output = self._wrap_code(output)
-        
+
         output = self._wrap_pre(output)
-    
+
         return output
 
     def format_unencoded(self, tokensource, outfile):

pygments/lexers/_lua_builtins.py

@@ -249,14 +249,14 @@ def get_function_module(name):
             return 'basic'
 
     def regenerate(filename, modules):
-        with open(filename) as fp:
+        with open(filename, encoding='utf-8') as fp:
             content = fp.read()
 
         header = content[:content.find('MODULES = {')]
         footer = content[content.find("if __name__ == '__main__':"):]
 
 
-        with open(filename, 'w') as fp:
+        with open(filename, 'w', encoding='utf-8') as fp:
             fp.write(header)
             fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
             fp.write(footer)

pygments/lexers/_mysql_builtins.py

@@ -1317,7 +1317,7 @@ def parse_item_create_functions(f):
     def update_content(field_name, content):
         """Overwrite this file with content parsed from MySQL's source code."""
 
-        with open(__file__) as f:
+        with open(__file__, encoding="utf-8") as f:
             data = f.read()
 
         # Line to start/end inserting
@@ -1329,7 +1329,7 @@ def update_content(field_name, content):
         new_block = format_lines(field_name, content)
         data = data[:m.start()] + new_block + data[m.end():]
 
-        with open(__file__, 'w', newline='\n') as f:
+        with open(__file__, 'w', encoding='utf-8', newline='\n') as f:
             f.write(data)
 
     update_myself()

pygments/lexers/_php_builtins.py

@@ -3267,7 +3267,7 @@ def get_php_functions():
 
         for file in get_php_references():
             module = ''
-            with open(file) as f:
+            with open(file, encoding='utf-8') as f:
                 for line in f:
                     if not module:
                         search = module_re.search(line)
@@ -3303,13 +3303,13 @@ def get_php_references():
         os.remove(download[0])
 
     def regenerate(filename, modules):
-        with open(filename) as fp:
+        with open(filename, encoding='utf-8') as fp:
             content = fp.read()
 
         header = content[:content.find('MODULES = {')]
         footer = content[content.find("if __name__ == '__main__':"):]
 
-        with open(filename, 'w') as fp:
+        with open(filename, 'w', encoding='utf-8') as fp:
             fp.write(header)
             fp.write('MODULES = %s\n\n' % pprint.pformat(modules))
             fp.write(footer)

pygments/lexers/_postgres_builtins.py

@@ -665,7 +665,7 @@ def parse_pseudos(f):
         return dt
 
     def update_consts(filename, constname, content):
-        with open(filename) as f:
+        with open(filename, encoding='utf-8') as f:
             data = f.read()
 
         # Line to start/end inserting
@@ -678,7 +678,7 @@ def update_consts(filename, constname, content):
         new_block = format_lines(constname, content)
         data = data[:m.start()] + new_block + data[m.end():]
 
-        with open(filename, 'w', newline='\n') as f:
+        with open(filename, 'w', encoding='utf-8', newline='\n') as f:
             f.write(data)
 
     update_myself()

pygments/lexers/_scilab_builtins.py

@@ -3079,13 +3079,13 @@ def extract_completion(var_type):
         seen.update(set(new_data[t]))
 
 
-    with open(__file__) as f:
+    with open(__file__, encoding='utf-8') as f:
         content = f.read()
 
     header = content[:content.find('# Autogenerated')]
     footer = content[content.find("if __name__ == '__main__':"):]
 
-    with open(__file__, 'w') as f:
+    with open(__file__, 'w', encoding='utf-8') as f:
         f.write(header)
         f.write('# Autogenerated\n\n')
         for k, v in sorted(new_data.items()):

pygments/lexers/_sourcemod_builtins.py

@@ -1122,14 +1122,14 @@ def get_sm_functions():
         return functions
 
     def regenerate(filename, natives):
-        with open(filename) as fp:
+        with open(filename, encoding='utf-8') as fp:
             content = fp.read()
 
         header = content[:content.find('FUNCTIONS = (')]
         footer = content[content.find("if __name__ == '__main__':")-1:]
 
 
-        with open(filename, 'w') as fp:
+        with open(filename, 'w', encoding='utf-8') as fp:
             fp.write(header)
             fp.write(format_lines('FUNCTIONS', natives))
             fp.write('\n\n' + footer)

pygments/unistring.py

@@ -112,7 +112,7 @@ def _handle_runs(char_list):  # pragma: no cover
 
     categories = {'xid_start': [], 'xid_continue': []}
 
-    with open(__file__) as fp:
+    with open(__file__, encoding='utf-8') as fp:
         content = fp.read()
 
     header = content[:content.find('Cc =')]
@@ -136,7 +136,7 @@ def _handle_runs(char_list):  # pragma: no cover
         if ('a' + c).isidentifier():
             categories['xid_continue'].append(c)
 
-    with open(__file__, 'w') as fp:
+    with open(__file__, 'w', encoding='utf-8') as fp:
         fp.write(header)
 
         for cat in sorted(categories):

scripts/get_css_properties.py

@@ -28,6 +28,6 @@
     data = json.load(data_request)
     names = set([p['property'] for p in data if p['property'] != '--*'])
 
-    with open('../pygments/lexers/_css_builtins.py', 'w') as builtin_file:
+    with open('../pygments/lexers/_css_builtins.py', 'w', encoding='utf-8') as builtin_file:
         builtin_file.write(HEADER)
         builtin_file.write(format_lines('_css_properties', sorted(names)))

scripts/utility.py

@@ -21,7 +21,7 @@ def unpack_output_file(path):
 
     skip_until_tokens = path.endswith('.txt')
 
-    for linenumber, line in enumerate(open(path).readlines()):
+    for linenumber, line in enumerate(open(path, encoding='utf-8').readlines()):
         line = line.strip()
         if not line:
             continue

tests/contrast/test_contrasts.py

@@ -60,7 +60,7 @@ def min_contrasts():
 
 
 def update_json():
-    with open(JSON_FILENAME, "w") as f:
+    with open(JSON_FILENAME, "w", encoding="utf-8") as f:
         json.dump(
             min_contrasts(),
             f,
@@ -69,7 +69,7 @@ def update_json():
 
 
 def test_contrasts(fail_if_improved=True):
-    with open(JSON_FILENAME) as f:
+    with open(JSON_FILENAME, encoding="utf-8") as f:
         previous_contrasts = json.load(f)
 
     for style_name in pygments.styles.STYLE_MAP:

tests/test_html_formatter_linenos_elements.py

@@ -57,7 +57,8 @@ def test_linenos_elements(
     #     import bs4 as BeautifulSoup
     #     f.write(str(BeautifulSoup.BeautifulSoup(html, 'html.parser')))
 
-    with open(os.path.join(EXPECTED_OUTPUT_DIR, expected_html_filename)) as f:
+    with open(os.path.join(EXPECTED_OUTPUT_DIR, expected_html_filename),
+              encoding="utf-8") as f:
         expected_html = f.read()
 
     structural_diff.structural_diff(html, expected_html)

tests/test_latex_formatter.py

@@ -24,7 +24,7 @@
 
 
 def test_correct_output():
-    with open(TESTFILE) as fp:
+    with open(TESTFILE, encoding='utf-8') as fp:
         tokensource = list(PythonLexer().get_tokens(fp.read()))
     hfmt = LatexFormatter(nowrap=True)
     houtfile = StringIO()
@@ -35,7 +35,7 @@ def test_correct_output():
 
 
 def test_valid_output():
-    with open(TESTFILE) as fp:
+    with open(TESTFILE, encoding='utf-8') as fp:
         tokensource = list(PythonLexer().get_tokens(fp.read()))
     fmt = LatexFormatter(full=True, encoding='latin1')