From 5c855faa3e2073e35f448ffe93363b4df082c48e Mon Sep 17 00:00:00 2001 From: dvanaken Date: Wed, 23 Oct 2019 15:59:22 -0400 Subject: [PATCH] Testing gitlint --- .gitlint.yaml | 400 +++++++++++++++++++++ .travis.yml | 4 +- client/controller/build.gradle | 12 - docker/Dockerfile.base-ubuntu-18.04 | 2 +- script/formatting/config/google_checks.xml | 8 +- script/formatting/config/pylintrc | 295 ++++++--------- script/formatting/formatter.py | 18 +- script/validators/source_validator.py | 10 +- server/website/requirements.txt | 9 +- server/website/website/forms.py | 6 +- server/website/website/models.py | 16 +- 11 files changed, 539 insertions(+), 241 deletions(-) create mode 100644 .gitlint.yaml diff --git a/.gitlint.yaml b/.gitlint.yaml new file mode 100644 index 0000000..353baf8 --- /dev/null +++ b/.gitlint.yaml @@ -0,0 +1,400 @@ +# Copyright 2013-2014 Sebastian Kreft +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Regular expression matchers like \d, \w, must be escaped as in \\d, \\w. +# If you need to include a string like '{}' or '{foo}', you need to double the +# braces, as in '{{}}' or '{{foo}}'. See the pylint configuration for an +# example. + +# NOTE: filter are regular expressions, and for readability they are broken up +# using '>-' line folding from YAML. This means that between each line a space +# will be added. + + +# Python +pylint: + extensions: + - .py + command: pylint + arguments: + - --rcfile={REPO_HOME}/script/formatting/config/pylintrc + - --output-format=text + - >- + --msg-template={{abspath}}:{{line}}:{{column}}: + [{{category}}:{{symbol}}] {{obj}}: {{msg}} + - --reports=n + - --verbose + filter: >- + ^{filename}:(?P{lines}):((?P\d+):)? + \[(?P.+):(?P\S+)\]\s+(: + )?(?P.+)$ + installation: "Run pip install pylint." + +# Sample output: +# package/module.py:68:80: E501 line too long (80 > 79 characters) +pycodestyle: + extensions: + - .py + command: pycodestyle + arguments: + - "--ignore=E121,E123,E126,E226,E24,E704,W503,E501" + - "--max-line-length=100" + filter: >- + ^{filename}:(?P{lines}):((?P\d+):)? + (?P\S+) (?P.+)$ + installation: "Run pip install pycodestyle." + +# Java +# Sample output: +# /path_to/error.java:0: Missing package-info.java file. +# /path_to/error.java:1:7: warning: Name 'foo' must match pattern '^[A-Z][a-zA-Z0-9]*$'. +checkstyle: + command: checkstyle + extensions: + - .java + requirements: + - java + arguments: + - -c + - "{REPO_HOME}/script/formatting/config/google_checks.xml" + filter: "{filename}:(?P{lines}):((?P\\d+):)? (?P.+)" + installation: >- + sudo apt-get install checkstyle or go to + http://checkstyle.sourceforge.net/cmdline.html + +## Sample output: +## /path_to/error.java:1: All methods are static. +## Disabled rulesets because of false positives +## rulesets/java/coupling.xml: Demeter +## rulesets/java/design.xml: Static class +## rulesets/java/optimizations.xml: Parameter could be final +## rulesets/java/junit.xml: maximum asserts, asserts should have message +#pmd: +# command: run.sh +# extensions: +# - .java +# requirements: +# - java +# arguments: +# - pmd +# - -format +# - text +# - -rulesets +# - "rulesets/java/android.xml,rulesets/java/basic.xml,\ +# rulesets/java/braces.xml,rulesets/java/clone.xml,\ +# rulesets/java/codesize.xml,rulesets/java/empty.xml,\ +# rulesets/java/finalizers.xml,rulesets/java/imports.xml,\ +# rulesets/java/j2ee.xml,rulesets/java/logging-jakarta-commons.xml,\ +# rulesets/java/strictexception.xml,rulesets/java/strings.xml,\ +# rulesets/java/sunsecure.xml,rulesets/java/typeresolution.xml,\ +# rulesets/java/unnecessary.xml,rulesets/java/unusedcode.xml" +# - -d +# filter: "{filename}:(?P{lines}):\\s+(?P.+)" +# installation: Go to http://pmd.sourceforge.net/pmd-5.1.1/installing.html +# +## JSON +## Sample output: +## Expecting property name: line 3 column 5 (char 15) +#json: +# extensions: +# - .json +# command: python +# arguments: +# - -m +# - json.tool +# # enforce that here comes a colon +# filter: >- +# ^(?P[^:]+(?=: +# line \d+ column \d+)|No JSON object could be decoded)(: +# line (?P\d+) column (?P\d+).*)?$ +# installation: Nothing else should be required. +# +## CSS +## Sample output: +## /path_to/error.css: line 3, col 2, Warning - Duplicate property 'width' found. +#csslint: +# extensions: +# - .css +# command: csslint +# arguments: +# - "--ignore=ids,box-model,adjoining-classes,qualified-headings,\ +# unique-headings,zero-units" +# - --format=compact +# filter: >- +# ^{filename}: line (?P{lines}), col (?P\d+)?, +# (?P\S+) - (?P.+) +# installation: >- +# Go to https://github.com/stubbornella/csslint/wiki/Command-line-interface +# for installation instructions. +# +## SCSS +## Sample output: +## /path_to/error.scss:2 [W] `0px` should be written without units as `0` +#scss: +# extensions: +# - .scss +# command: scss-lint +# filter: >- +# ^{filename}:(?P{lines})(:(?P\d+))? +# \[(?P.+)\]( (?P.+):)? +# (?P.+) +# installation: >- +# gem install scss-lint or go to https://github.com/causes/scss-lint +# +## Javascript +## Sample output: +## Line 1, E:0002: Missing space before "=" +#gjslint: +# extensions: +# - .js +# command: gjslint +# filter: >- +# ^Line\s+(?P{lines}), (?P[^: ]+):((?P\d+):)? +# (?P.+) +# installation: >- +# Run pip install +# http://closure-linter.googlecode.com/files/closure_linter-latest.tar.gz, +# or visit https://developers.google.com/closure/utilities/docs/linter_howto +# for installation instructions. +# +## Sample output: +## /path_to/error.js: line 1, col 3, Use '===' to compare with ''. +#jshint: +# extensions: +# - .js +# command: jshint +# arguments: +# - --config +# - "{DEFAULT_CONFIGS}/jshint.json" +# filter: >- +# ^{filename}: line (?P{lines}), col (?P\d+), (?P.+) +# installation: >- +# Visit http://www.jshint.com/install/ for installation instructions. +# +## PHP +## Sample output: +## PHP Parse error: syntax error, unexpected 'bar' (T_STRING) in /path_to/error.php on line 3 +#php: +# extensions: +# - .php +# command: php +# arguments: +# - -l +# filter: ^(?P.*) in {filename} on line (?P\d+) +# installation: You first need to install PHP. +# +## Sample output: +## 2 | ERROR | Expected "if (...) {\n"; found "if (...) {" +#phpcs: +# extensions: +# - .php +# command: phpcs +# arguments: +# - --report-width=1000 +# - --standard=PSR2 +# filter: >- +# ^\s*(?P{lines})\s+[|]\s+(?P\S+)\s+[|]\s+(?P.+) +# installation: >- +# Visit https://github.com/squizlabs/PHP_CodeSniffer for installation +# instructions +# +# +## RST +## Sample output: +## /path_to/error.rst:3: (WARNING/2) Inline interpreted text or phrase reference start-string without end-string. +#rst: +# extensions: +# - .rst +# command: rst2html.py +# filter: >- +# ^{filename}:(?P{lines}): [(](?P.+)[)] (?P.+) +# installation: Run pip install docutils. +# +## PNG +#pngcrush: +# extensions: +# - .png +# command: pngcrush-linter.sh +# requirements: +# - pngcrush +# filter: (?P.+)$ +# installation: Run apt-get install pngcrush. +# +#optipng: +# extensions: +# - .png +# command: optipng-linter.sh +# requirements: +# - optipng +# filter: (?P.+)$ +# installation: Run apt-get install optipng. +# +## JPEG +#jpegtran: +# extensions: +# - .jpg +# - .jpeg +# command: jpegtran-linter.sh +# requirements: +# - jpegtran +# filter: (?P.+) +# installation: Run apt-get install jpegtran. +# +## SHELL scripts +## Sample output +## /path_to/error.sh: line 3: syntax error: unexpected end of file +#bash: +# extensions: +# - .sh +# command: bash +# arguments: +# - "-n" +# filter: >- +# {filename}: line (?P{lines}): (?P.+) +# installation: Please install bash in your system. +# +## YAML +#yaml: +# extensions: +# - .yaml +# - .yml +# command: yamllint +# arguments: +# - --format +# - parsable +# - --config-data +# - "{{extends: default, rules: {{document-start: disable}}}}" +# # Matches either: +# # - syntax error, on any line +# # - other error, on a modified line only +# filter: >- +# ^{filename}:(?P{lines}|\d+(?=:\d+: +# \[error\] syntax error:)):(?P\d+): +# \[(?P\S+)\] (?P.+)$ +# +# installation: Run pip install yamllint. +# +## INI +#ini: +# extensions: +# - .ini +# - .cfg +# command: ini_linter.py +# filter: (?P.+)$ +# installation: "" +# +## HTML +## Sample output: +## line 2 column 1 - Warning: missing before +#tidy: +# extensions: +# - .html +# command: tidy-wrapper.sh +# requirements: +# - tidy +# - remove_template.py +# - grep +# arguments: +# - -qe +# - --drop-empty-elements +# - "false" +# installation: Visit https://w3c.github.io/tidy-html5/ +# filter: >- +# ^line (?P{lines}) column (?P\d+) - +# (?P[^:]+): (?P.+) +# +## Sample output: +## 1:10: Error: Javascript ... +#html_lint: +# extensions: +# - .html +# command: html_lint.py +# arguments: +# - --disable +# - optional_tag +# installation: pip install html-linter. +# filter: >- +# ^(?P{lines}):(?P\d+): (?P\S+): (?P.+) +# +## Ruby +## Sample output: +## error.rb: warning: line 1, column 1: unused constant FOO +#rubylint: +# command: ruby-lint +# arguments: +# - --analysis +# - "argument_amount,loop_keywords,pedantics,shadowing_variables,\ +# unused_variables,useless_equality_checks" +# extensions: +# - .rb +# # The first component is the basename, but it's not supported yet. +# filter: >- +# .*: (?P.+): line (?P{lines}), +# column (?P\d+): (?P.+) +# installation: >- +# sudo gem install ruby-lint (requires ruby 1.9) or visit +# https://github.com/yorickpeterse/ruby-lint +# +# +## Sample output with the --format emacs option: +## /path_to/error.rb:1:4: C: Surrounding space missing for operator '='. +#rubocop: +# command: rubocop +# arguments: +# - --format +# - emacs +# - --rails +# extensions: +# - .rb +# # The first component is the relpath, but it's not supported yet. +# filter: >- +# {filename}:(?P{lines}):(?P\d+): +# (?P.+): (?P.+) +# installation: >- +# sudo gem install rubocop or visit https://github.com/bbatsov/rubocop +# +## Coffeescript +## Sample output: +## /path_to/error.coffee,4,,error,Operators must be spaced properly =. +## /path_to/error.coffee,5,,error,[stdin]:5:1: error: reserved word 'yes' can't be assigned +#coffeelint: +# command: coffeelint +# extensions: +# - .coffee +# arguments: +# - --reporter=csv +# - --file={DEFAULT_CONFIGS}/coffeelint.json +# filter: >- +# {filename},(?P{lines}),.*,(?P.+),(?:\[stdin\]:\d+:(?P\d+): .*: )?(?P.+) +# installation: npm install -g coffeelint +# +## C++ +## Sample output: +## main.cpp:0: No copyright message found. You should have a line: "Copyright [year] " [legal/copyright] [5] +#cpplint: +# extensions: +# - .c +# - .cc +# - .cpp +# - .c++ +# - .h +# - .hpp +# - .h++ +# command: cpplint +# requirements: +# - cpplint +# filter: >- +# ^{filename}:(?P{lines}): (?P.+) \[(?P.+)\] +# \[(?P\d+)\] +# installation: "Run pip install cpplint." diff --git a/.travis.yml b/.travis.yml index 525cfd1..00ca40f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,9 +45,9 @@ script: - docker-compose -f docker-compose.test.yml run --workdir="/app/client/controller" --rm test gradle build - docker-compose -f docker-compose.test.yml run --workdir="/app/server" --rm $ci_env test bash -c "coverage run --omit=\"*/tests/*\" -m unittest discover -s analysis/tests -v && (codecov -F analysis || (sleep 5 && codecov -F analysis) || (sleep 5 && codecov -F analysis))" - docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm $ci_env test bash -c "./wait-for-it.sh && python3 manage.py makemigrations website && coverage run --source=website manage.py test --noinput -v 2 && (codecov -F website || (sleep 5 && codecov -F website) || (sleep 5 && codecov -F website))" - # Only run source validation once (on ubuntu 18.04) + # Only run the linter once on ubuntu 18.04 - if [ $DOCKER_OS == ubuntu-18.04 ]; then - docker-compose -f docker-compose.test.yml run --workdir="/app" --rm test python3 script/validators/source_validator.py; + docker-compose -f docker-compose.test.yml run --workdir="/app" --rm test bash -c "git reset --soft ${TRAVIS_COMMIT_RANGE%...*} && git lint"; fi after_script: diff --git a/client/controller/build.gradle b/client/controller/build.gradle index ae3c70e..703db99 100644 --- a/client/controller/build.gradle +++ b/client/controller/build.gradle @@ -1,5 +1,4 @@ plugins { - id "de.undercouch.download" version "3.3.0" id "com.github.spotbugs" version "2.0.0" } @@ -77,14 +76,3 @@ tasks.withType(com.github.spotbugs.SpotBugsTask) { } } -import de.undercouch.gradle.tasks.download.Download -task downloadJars(type: Download) { - src ([ - 'https://github.com/google/google-java-format/releases/download/google-java-format-1.5/google-java-format-1.5-all-deps.jar', - 'https://github.com/checkstyle/checkstyle/releases/download/checkstyle-8.8/checkstyle-8.8-all.jar' - ]) - dest libsDir - overwrite false -} - -build.finalizedBy(downloadJars) diff --git a/docker/Dockerfile.base-ubuntu-18.04 b/docker/Dockerfile.base-ubuntu-18.04 index e2f5406..acc2a0b 100644 --- a/docker/Dockerfile.base-ubuntu-18.04 +++ b/docker/Dockerfile.base-ubuntu-18.04 @@ -11,7 +11,7 @@ COPY ./server/website/requirements.txt / RUN apt-get update \ && apt-get install -y python3.6 python3-pip mysql-client \ libmysqlclient-dev python-mysqldb postgresql-client \ - openjdk-11-jdk git unzip wget curl \ + openjdk-11-jdk checkstyle git unzip wget curl \ && wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \ && unzip ${GRADLE_VERSION}-bin.zip -d /opt \ && rm ${GRADLE_VERSION}-bin.zip \ diff --git a/script/formatting/config/google_checks.xml b/script/formatting/config/google_checks.xml index a8ec132..7d8a25d 100644 --- a/script/formatting/config/google_checks.xml +++ b/script/formatting/config/google_checks.xml @@ -26,6 +26,10 @@ + + + + @@ -39,10 +43,6 @@ - - - - diff --git a/script/formatting/config/pylintrc b/script/formatting/config/pylintrc index 66705fa..e73ffcd 100644 --- a/script/formatting/config/pylintrc +++ b/script/formatting/config/pylintrc @@ -3,48 +3,28 @@ # Specify a configuration file. #rcfile= -# Python code to execute. Adds the analysis directory to the system path -# variable which is necessary to avoid ImportErrors in our website code. -init-hook='import os, sys; cwd = os.getcwd(); analysis_path = os.path.join(cwd, 'server') if '/server/' not in cwd else cwd[0:cwd.index('/server/') + len('/server/') - 1]; sys.path.insert(0, analysis_path)' +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +# init-hook='import sys; import os; sys.path.append(os.path.abspath("."))' +init-hook='import sys; sys.path.append("."); sys.path.append("./server")' + +# Profiled execution. +profile=no # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=CVS +ignore=CVS,.git # Pickle collected data for later comparisons. -persistent=yes +persistent=no # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= -# Use multiple processes to speed up Pylint. -jobs=1 - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Allow optimization of some AST trees. This will activate a peephole AST -# optimizer, which will apply various small optimizations. For instance, it can -# be used to obtain the result of joining multiple strings with the addition -# operator. Joining a lot of strings can lead to a maximum recursion error in -# Pylint and this flag can prevent that. It has one side effect, the resulting -# AST will be different than the one from reality. -optimize-ast=no - [MESSAGES CONTROL] -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence=HIGH,INFERENCE_FAILURE,UNDEFINED - # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. See also the "--disable" option for examples. @@ -59,6 +39,7 @@ confidence=HIGH,INFERENCE_FAILURE,UNDEFINED # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" +# Custom setting disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,missing-docstring,too-few-public-methods,too-many-arguments,too-many-locals,too-many-instance-attributes,too-many-statements,locally-disabled,superfluous-parens,too-many-branches,not-callable,too-many-nested-blocks,fixme,redefined-variable-type,no-member,locally-enabled,too-many-public-methods @@ -84,11 +65,33 @@ reports=no # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + # Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details +# used to format the massage information. See doc for all details #msg-template= +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=100 + +# Regexp for a line that is allowed to be longer than the limit. +# We also allow declaration of url in string to be longer as it is useful for +# automatic linking. +ignore-long-lines=^\s*(# )??$|'https?://\S+'$ + +# Maximum number of lines in a module +max-module-lines=2000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + [SIMILARITIES] # Minimum lines number of a similarity. @@ -112,43 +115,21 @@ ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= +# and thus existing member attributes cannot be deduced by static analysis +ignored-modules=numpy # List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). This supports can work -# with qualified names. -ignored-classes= +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject,pytest + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no # List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular +# system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. -generated-members= - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no +generated-members=REQUEST,acl_users,aq_parent [VARIABLES] @@ -156,49 +137,71 @@ spelling-store-unknown-words=no # Tells whether we should check for unused import in __init__ files. init-import=no -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_$|dummy|unused_ # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb +[BASIC] -[FORMAT] +# Required attributes for module, separated by a comma +required-attributes= -# Maximum number of characters on a single line. -max-line-length=100 +# List of builtins function names that should not be used, separated by a comma +bad-functions=filter,apply,input -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no +# Regular expression which should only match correct module level names +const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|logger)$ -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ -# Maximum number of lines in a module -max-module-lines=1000 +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-z0-9_]{2,30}$|test_[a-z0-9_]{2,50}$ -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-z0-9_]{2,30}$|test_[a-z0-9_]{2,50}$ -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 +# Regular expression which should only match correct instance attribute names +# attr-rgx=[a-z_][a-z0-9_]{2,30}$ +# Custom setting +attr-rgx=(([a-z_][a-z0-9_]{1,30})|([XK][a-z0-9_]{0,30}))$ -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= +# Regular expression which should only match correct argument names +argument-rgx=[a-zX_][a-z0-9_]{0,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-zX_][a-z0-9_]{0,30}$ + +# Regular expression which should only match correct attribute names in class +# bodies +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +# Custom setting +good-names=f,i,j,k,ex,Run,_,mu,y + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=__.*__|.*Test|test_.*|_.* + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 [MISCELLANEOUS] @@ -207,101 +210,12 @@ expected-line-ending-format= notes=FIXME,XXX,TODO -[BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,input - -# Good variable names which should always be accepted, separated by a comma -good-names=ex,Run,_,mu,y - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{0,30}$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z_][a-z0-9_]{1,30})|([XK][a-z0-9_]{0,30}))$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{1,30}$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[ELIF] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - [CLASSES] +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp @@ -311,10 +225,6 @@ valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - [IMPORTS] @@ -337,7 +247,7 @@ int-import-graph= [DESIGN] # Maximum number of arguments for function / method -max-args=5 +max-args=8 # Argument names that match this expression will be ignored. Default to name # with leading underscore @@ -350,7 +260,7 @@ max-locals=15 max-returns=6 # Maximum number of branch for function / method body -max-branches=12 +max-branches=18 # Maximum number of statements in function / method body max-statements=50 @@ -359,17 +269,14 @@ max-statements=50 max-parents=7 # Maximum number of attributes for a class (see R0902). -max-attributes=7 +max-attributes=15 # Minimum number of public methods for a class (see R0903). -min-public-methods=2 +min-public-methods=1 # Maximum number of public methods for a class (see R0904). max-public-methods=20 -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - [EXCEPTIONS] diff --git a/script/formatting/formatter.py b/script/formatting/formatter.py index 439c77f..86508a3 100644 --- a/script/formatting/formatter.py +++ b/script/formatting/formatter.py @@ -12,6 +12,7 @@ import subprocess import sys import autopep8 +from fabric.api import local EXIT_SUCCESS = 0 EXIT_FAILURE = -1 @@ -44,8 +45,10 @@ OTTERTUNE_DIR = os.path.abspath(functools.reduce(os.path.join, os.path.pardir, os.path.pardir])) +JAVA_JAR_VERSION = '1.5' JAVA_JAR_PATH = os.path.join( - OTTERTUNE_DIR, 'controller/build/libs/google-java-format-1.5-all-deps.jar') + OTTERTUNE_DIR, 'script/formatting/config', + 'google-java-format-{}-all-deps.jar'.format(JAVA_JAR_VERSION)) # ============================================== # FILE HEADER FORMATS @@ -113,9 +116,16 @@ def format_java_file(file_path, update_header, format_code): if format_code: if not os.path.exists(JAVA_JAR_PATH): - controller_dir = os.path.join(OTTERTUNE_DIR, 'controller') - subprocess.check_output(["gradle", "downloadJars"], cwd=controller_dir) - subprocess.check_output(["java", "-jar", JAVA_JAR_PATH, "-r", file_path]) + local(( + 'wget https://github.com/google/google-java-format/' + 'releases/download/google-java-format-{0}/' + 'google-java-format-{0}-all-deps.jar && ' + 'mv {1} {2}').format( + JAVA_JAR_VERSION, + os.path.basename(JAVA_JAR_PATH), + JAVA_JAR_PATH)) + + local('java -jar {} -r {}'.format(JAVA_JAR_PATH, file_path)) def format_python_file(file_path, update_header, format_code): diff --git a/script/validators/source_validator.py b/script/validators/source_validator.py index c803a5b..ca7d233 100644 --- a/script/validators/source_validator.py +++ b/script/validators/source_validator.py @@ -79,9 +79,6 @@ EXCLUDE_FILES = [ os.path.join(OTTERTUNE_DIR, 'server/analysis/simulation.py'), ] -CHECKSTYLE_JAR_PATH = os.path.join(OTTERTUNE_DIR, - "client/controller/build/libs/checkstyle-8.8-all.jar") - # Regex patterns PYCODESTYLE_COMMENT_PATTERN = re.compile(r'#\s*pycodestyle:\s*disable\s*=\s*[\w\,\s]+$') @@ -326,14 +323,9 @@ def check_java_checkstyle(file_path, config_path=None): if not file_path.endswith(".java"): return True, None - if not os.path.exists(CHECKSTYLE_JAR_PATH): - with lcd(os.path.join(OTTERTUNE_DIR, "client/controller")): # pylint: disable=not-context-manager - local("gradle downloadJars") - options = '' if config_path is None else '-c ' + config_path with quiet(): - res = local("java -jar {} {} {}".format(CHECKSTYLE_JAR_PATH, options, file_path), - capture=True) + res = local("checkstyle {} {}".format(options, file_path), capture=True) lines = res.stdout.split('\n') assert len(lines) >= 2 and lines[0] == "Starting audit..." and lines[-1] == "Audit done." if len(lines) == 2: diff --git a/server/website/requirements.txt b/server/website/requirements.txt index b7a3d5c..6593ae7 100644 --- a/server/website/requirements.txt +++ b/server/website/requirements.txt @@ -1,4 +1,4 @@ -autopep8==1.3.4 +autopep8==1.4.4 celery==3.1.23 Django==1.11.23 django-celery==3.2.1 @@ -7,13 +7,14 @@ django-db-logger>=0.1.7 django-request-logging==0.4.6 mock==2.0.0 Fabric3>=1.13.1.post1 +git-lint==0.1.2 hurry.filesize>=0.9 numpy==1.14.0 requests==2.20.0 -pycodestyle==2.3.1 -astroid==1.5.1 +pycodestyle==2.5.0 +astroid==2.3.2 psycopg2-binary>=2.5.4 -pylint==1.5.2 +pylint==2.4.3 pyDOE>=0.3.8 mysqlclient==1.3.12 scikit-learn==0.19.1 diff --git a/server/website/website/forms.py b/server/website/website/forms.py index ee21cdd..c10ef3a 100644 --- a/server/website/website/forms.py +++ b/server/website/website/forms.py @@ -39,7 +39,7 @@ class ProjectForm(forms.ModelForm): self._errors['name'] = ["Project '{}' already exists.".format(new_name)] return valid - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init model = Project fields = ['name', 'description'] @@ -130,7 +130,7 @@ class SessionForm(forms.ModelForm): return model - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init model = Session fields = ('name', 'description', 'tuning_session', 'dbms', 'cpu', 'memory', 'storage', @@ -154,6 +154,6 @@ class SessionKnobForm(forms.ModelForm): self.fields['knob'].required = False self.fields['name'].widget.attrs['readonly'] = True - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init model = SessionKnob fields = ['session', 'knob', 'minval', 'maxval', 'tunable'] diff --git a/server/website/website/models.py b/server/website/website/models.py index b75be27..01db1a7 100644 --- a/server/website/website/models.py +++ b/server/website/website/models.py @@ -43,7 +43,7 @@ class BaseModel(models.Model): def _model_name(cls): return cls.__name__ - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init abstract = True @@ -107,7 +107,7 @@ class Project(BaseModel): x.delete() super(Project, self).delete(using, keep_parents) - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init unique_together = ('user', 'name') @@ -125,7 +125,7 @@ class Hardware(BaseModel): default=StorageType.SSD, verbose_name='Storage Type') additional_specs = models.TextField(null=True, default=None) - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init unique_together = ('cpu', 'memory', 'storage', 'storage_type') @@ -176,7 +176,7 @@ class Session(BaseModel): r.delete() super(Session, self).delete(using=DEFAULT_DB_ALIAS, keep_parents=False) - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init unique_together = ('user', 'project', 'name') @@ -243,7 +243,7 @@ class DataModel(BaseModel): data = models.TextField() dbms = models.ForeignKey(DBMSCatalog) - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init abstract = True @@ -334,7 +334,7 @@ class Workload(BaseModel): x.delete() super(Workload, self).delete(using, keep_parents) - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init unique_together = ("dbms", "hardware", "name") # @property @@ -413,7 +413,7 @@ class PipelineRun(models.Model): def __str__(self): return self.__unicode__() - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init ordering = ["-id"] @@ -424,7 +424,7 @@ class PipelineData(models.Model): data = models.TextField() creation_time = models.DateTimeField() - class Meta: # pylint: disable=old-style-class,no-init + class Meta: # pylint: disable=no-init unique_together = ("pipeline_run", "task_type", "workload")