From cbb143e9f1396c8180401c4f8a5b1fe579902559 Mon Sep 17 00:00:00 2001 From: Alexandru Croitor Date: Fri, 24 May 2019 17:29:21 +0200 Subject: Improve configurejson2cmake.py to handle non-compliant qmake JSON Some configure.json files contain new lines inside quoted strings, which is not conformant with the JSON spec. Add a new json_parser python module which uses pyparsing to preprocess the json files to remove the new lines inside the quoted strings, and then hands over the preprocessed content to the regular json module. Change-Id: I5f8938492068dda5640465cc78f5a7b6be0e709a Reviewed-by: Simon Hausmann Reviewed-by: Qt CMake Build Bot --- util/cmake/json_parser.py | 100 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 util/cmake/json_parser.py (limited to 'util/cmake/json_parser.py') diff --git a/util/cmake/json_parser.py b/util/cmake/json_parser.py new file mode 100644 index 00000000000..6ead008f081 --- /dev/null +++ b/util/cmake/json_parser.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +############################################################################# +## +## Copyright (C) 2019 The Qt Company Ltd. +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the plugins of the Qt Toolkit. +## +## $QT_BEGIN_LICENSE:GPL-EXCEPT$ +## Commercial License Usage +## Licensees holding valid commercial Qt licenses may use this file in +## accordance with the commercial license agreement provided with the +## Software or, alternatively, in accordance with the terms contained in +## a written agreement between you and The Qt Company. For licensing terms +## and conditions see https://www.qt.io/terms-conditions. For further +## information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 as published by the Free Software +## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +############################################################################# + +import pyparsing as pp +import json +import re +from helper import _set_up_py_parsing_nicer_debug_output +_set_up_py_parsing_nicer_debug_output(pp) + + +class QMakeSpecificJSONParser: + def __init__(self, *, debug: bool = False) -> None: + self.debug = debug + self.grammar = self.create_py_parsing_grammar() + + def create_py_parsing_grammar(self): + # Keep around all whitespace. + pp.ParserElement.setDefaultWhitespaceChars('') + + def add_element(name: str, value: pp.ParserElement): + nonlocal self + if self.debug: + value.setName(name) + value.setDebug() + return value + + # Our grammar is pretty simple. We want to remove all newlines + # inside quoted strings, to make the quoted strings JSON + # compliant. So our grammar should skip to the first quote while + # keeping everything before it as-is, process the quoted string + # skip to the next quote, and repeat that until the end of the + # file. + + EOF = add_element('EOF', pp.StringEnd()) + SkipToQuote = add_element('SkipToQuote', pp.SkipTo('"')) + SkipToEOF = add_element('SkipToEOF', pp.SkipTo(EOF)) + + def remove_newlines_and_whitespace_in_quoted_string(tokens): + first_string = tokens[0] + replaced_string = re.sub(r'\n[ ]*', ' ', first_string) + return replaced_string + + QuotedString = add_element('QuotedString', pp.QuotedString(quoteChar='"', + multiline=True, + unquoteResults=False)) + QuotedString.setParseAction(remove_newlines_and_whitespace_in_quoted_string) + + QuotedTerm = add_element('QuotedTerm', pp.Optional(SkipToQuote) + QuotedString) + Grammar = add_element('Grammar', pp.OneOrMore(QuotedTerm) + SkipToEOF) + + return Grammar + + def parse_file_using_py_parsing(self, file: str): + print('Pre processing "{}" using py parsing to remove incorrect newlines.'.format(file)) + try: + with open(file, 'r') as file_fd: + contents = file_fd.read() + + parser_result = self.grammar.parseString(contents, parseAll=True) + token_list = parser_result.asList() + joined_string = ''.join(token_list) + + return joined_string + except pp.ParseException as pe: + print(pe.line) + print(' '*(pe.col-1) + '^') + print(pe) + raise pe + + def parse(self, file: str): + pre_processed_string = self.parse_file_using_py_parsing(file) + print('Parsing "{}" using json.loads().'.format(file)) + json_parsed = json.loads(pre_processed_string) + return json_parsed -- cgit v1.2.3 From 91634c3c9b8d4f68f0ebd2ac76a8b5b79e4b4c94 Mon Sep 17 00:00:00 2001 From: Cristian Maureira-Fredes Date: Tue, 17 Sep 2019 00:11:17 +0200 Subject: Improve styling of util/cmake scripts flake8 was used to evaluate the file, with a couple of exeptions: E501,E266,W503 black was used to reformat the code automatically The changes were: * Added a README that explains how to use pipenv and pip, * Remove unnecessary return statements, * Remove '\' from the end of the lines, * Use f-strings (>= 3.6) since we are requiring Python 3.7, * Commenting unused variables, * Adding assert when Python >= 3.7 is not being used, * Wrapping long lines to 100 (Qt Style), * Re-factoring some lines, * Re-ordering imports, * Naming `except` for sympy (SympifyError, TypeError) Change-Id: Ie05f754e7d8ee4bf427117c58e0eb1b903202933 Reviewed-by: Alexandru Croitor --- util/cmake/json_parser.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'util/cmake/json_parser.py') diff --git a/util/cmake/json_parser.py b/util/cmake/json_parser.py index 6ead008f081..36e8cee37ff 100644 --- a/util/cmake/json_parser.py +++ b/util/cmake/json_parser.py @@ -31,6 +31,7 @@ import pyparsing as pp import json import re from helper import _set_up_py_parsing_nicer_debug_output + _set_up_py_parsing_nicer_debug_output(pp) @@ -41,7 +42,7 @@ class QMakeSpecificJSONParser: def create_py_parsing_grammar(self): # Keep around all whitespace. - pp.ParserElement.setDefaultWhitespaceChars('') + pp.ParserElement.setDefaultWhitespaceChars("") def add_element(name: str, value: pp.ParserElement): nonlocal self @@ -57,44 +58,44 @@ class QMakeSpecificJSONParser: # skip to the next quote, and repeat that until the end of the # file. - EOF = add_element('EOF', pp.StringEnd()) - SkipToQuote = add_element('SkipToQuote', pp.SkipTo('"')) - SkipToEOF = add_element('SkipToEOF', pp.SkipTo(EOF)) + EOF = add_element("EOF", pp.StringEnd()) + SkipToQuote = add_element("SkipToQuote", pp.SkipTo('"')) + SkipToEOF = add_element("SkipToEOF", pp.SkipTo(EOF)) def remove_newlines_and_whitespace_in_quoted_string(tokens): first_string = tokens[0] - replaced_string = re.sub(r'\n[ ]*', ' ', first_string) + replaced_string = re.sub(r"\n[ ]*", " ", first_string) return replaced_string - QuotedString = add_element('QuotedString', pp.QuotedString(quoteChar='"', - multiline=True, - unquoteResults=False)) + QuotedString = add_element( + "QuotedString", pp.QuotedString(quoteChar='"', multiline=True, unquoteResults=False) + ) QuotedString.setParseAction(remove_newlines_and_whitespace_in_quoted_string) - QuotedTerm = add_element('QuotedTerm', pp.Optional(SkipToQuote) + QuotedString) - Grammar = add_element('Grammar', pp.OneOrMore(QuotedTerm) + SkipToEOF) + QuotedTerm = add_element("QuotedTerm", pp.Optional(SkipToQuote) + QuotedString) + Grammar = add_element("Grammar", pp.OneOrMore(QuotedTerm) + SkipToEOF) return Grammar def parse_file_using_py_parsing(self, file: str): - print('Pre processing "{}" using py parsing to remove incorrect newlines.'.format(file)) + print(f'Pre processing "{file}" using py parsing to remove incorrect newlines.') try: - with open(file, 'r') as file_fd: + with open(file, "r") as file_fd: contents = file_fd.read() parser_result = self.grammar.parseString(contents, parseAll=True) token_list = parser_result.asList() - joined_string = ''.join(token_list) + joined_string = "".join(token_list) return joined_string except pp.ParseException as pe: print(pe.line) - print(' '*(pe.col-1) + '^') + print(" " * (pe.col - 1) + "^") print(pe) raise pe def parse(self, file: str): pre_processed_string = self.parse_file_using_py_parsing(file) - print('Parsing "{}" using json.loads().'.format(file)) + print(f'Parsing "{file}" using json.loads().') json_parsed = json.loads(pre_processed_string) return json_parsed -- cgit v1.2.3 From f95988261631a47fcc5cacf57a1226cb8e391c64 Mon Sep 17 00:00:00 2001 From: Frederik Gladhorn Date: Tue, 8 Oct 2019 15:36:05 +0200 Subject: cmake scripts: more type cleanup Change-Id: Ic32394548bb997af96756f260b453e830d8b9e9b Reviewed-by: Alexandru Croitor --- util/cmake/json_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'util/cmake/json_parser.py') diff --git a/util/cmake/json_parser.py b/util/cmake/json_parser.py index 36e8cee37ff..a0aaecab9da 100644 --- a/util/cmake/json_parser.py +++ b/util/cmake/json_parser.py @@ -27,7 +27,7 @@ ## ############################################################################# -import pyparsing as pp +import pyparsing as pp # type: ignore import json import re from helper import _set_up_py_parsing_nicer_debug_output -- cgit v1.2.3