public/ottertune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit with BSL

Browse Source
This commit is contained in:
Andy Pavlo
2019-08-23 11:47:19 -04:00
commit 3e564ce922
286 changed files with 177642 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

238
script/formatting/config/google_checks.xml Normal file
View File

@@ -0,0 +1,238 @@
<?xml version="1.0"?>
<!DOCTYPE module PUBLIC
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
<!--
Checkstyle configuration that checks the Google coding conventions from Google Java Style
that can be found at https://google.github.io/styleguide/javaguide.html.
Checkstyle is very configurable. Be sure to read the documentation at
http://checkstyle.sf.net (or in your downloaded distribution).
To completely disable a check, just comment it out or delete it from the file.
Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
-->
<module name = "Checker">
<property name="charset" value="UTF-8"/>
<property name="severity" value="warning"/>
<property name="fileExtensions" value="java, properties, xml"/>
<!-- Checks for whitespace -->
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
<module name="FileTabCharacter">
<property name="eachLine" value="true"/>
</module>
<module name="TreeWalker">
<module name="OuterTypeFilename"/>
<module name="IllegalTokenText">
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
</module>
<module name="AvoidEscapedUnicodeCharacters">
<property name="allowEscapesForControlCharacters" value="true"/>
<property name="allowByTailComment" value="true"/>
<property name="allowNonPrintableEscapes" value="true"/>
</module>
<module name="LineLength">
<property name="max" value="100"/>
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
</module>
<module name="AvoidStarImport"/>
<module name="OneTopLevelClass"/>
<module name="NoLineWrap"/>
<module name="EmptyBlock">
<property name="option" value="TEXT"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
</module>
<module name="NeedBraces"/>
<module name="LeftCurly"/>
<module name="RightCurly">
<property name="id" value="RightCurlySame"/>
<property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/>
</module>
<module name="RightCurly">
<property name="id" value="RightCurlyAlone"/>
<property name="option" value="alone"/>
<property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
</module>
<module name="WhitespaceAround">
<property name="allowEmptyConstructors" value="true"/>
<property name="allowEmptyMethods" value="true"/>
<property name="allowEmptyTypes" value="true"/>
<property name="allowEmptyLoops" value="true"/>
<message key="ws.notFollowed"
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
<message key="ws.notPreceded"
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
</module>
<module name="OneStatementPerLine"/>
<module name="MultipleVariableDeclarations"/>
<module name="ArrayTypeStyle"/>
<module name="MissingSwitchDefault"/>
<module name="FallThrough"/>
<module name="UpperEll"/>
<module name="ModifierOrder"/>
<module name="EmptyLineSeparator">
<property name="allowNoEmptyLineBetweenFields" value="true"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapDot"/>
<property name="tokens" value="DOT"/>
<property name="option" value="nl"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapComma"/>
<property name="tokens" value="COMMA"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
<property name="id" value="SeparatorWrapEllipsis"/>
<property name="tokens" value="ELLIPSIS"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
<property name="id" value="SeparatorWrapArrayDeclarator"/>
<property name="tokens" value="ARRAY_DECLARATOR"/>
<property name="option" value="EOL"/>
</module>
<module name="SeparatorWrap">
<property name="id" value="SeparatorWrapMethodRef"/>
<property name="tokens" value="METHOD_REF"/>
<property name="option" value="nl"/>
</module>
<module name="PackageName">
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
<message key="name.invalidPattern"
value="Package name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="TypeName">
<message key="name.invalidPattern"
value="Type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MemberName">
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
<message key="name.invalidPattern"
value="Member name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern"
value="Parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="CatchParameterName">
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern"
value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="LocalVariableName">
<property name="tokens" value="VARIABLE_DEF"/>
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
<message key="name.invalidPattern"
value="Local variable name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="ClassTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Class type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="MethodTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Method type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="InterfaceTypeParameterName">
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
<message key="name.invalidPattern"
value="Interface type name ''{0}'' must match pattern ''{1}''."/>
</module>
<module name="NoFinalizer"/>
<module name="GenericWhitespace">
<message key="ws.followed"
value="GenericWhitespace ''{0}'' is followed by whitespace."/>
<message key="ws.preceded"
value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
<message key="ws.illegalFollow"
value="GenericWhitespace ''{0}'' should followed by whitespace."/>
<message key="ws.notPreceded"
value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
</module>
<module name="Indentation">
<property name="basicOffset" value="2"/>
<property name="braceAdjustment" value="0"/>
<property name="caseIndent" value="2"/>
<property name="throwsIndent" value="4"/>
<property name="lineWrappingIndentation" value="4"/>
<property name="arrayInitIndent" value="2"/>
</module>
<!-- <module name="AbbreviationAsWordInName">
<property name="ignoreFinal" value="false"/>
<property name="allowedAbbreviationLength" value="4"/>
</module> -->
<module name="OverloadMethodsDeclarationOrder"/>
<module name="VariableDeclarationUsageDistance"/>
<module name="CustomImportOrder">
<property name="sortImportsInGroupAlphabetically" value="true"/>
<property name="separateLineBetweenGroups" value="true"/>
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
</module>
<module name="MethodParamPad"/>
<module name="NoWhitespaceBefore">
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
<property name="allowLineBreaks" value="true"/>
</module>
<module name="ParenPad"/>
<module name="OperatorWrap">
<property name="option" value="NL"/>
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationMostCases"/>
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
</module>
<module name="AnnotationLocation">
<property name="id" value="AnnotationLocationVariables"/>
<property name="tokens" value="VARIABLE_DEF"/>
<property name="allowSamelineMultipleAnnotations" value="true"/>
</module>
<!-- <module name="NonEmptyAtclauseDescription"/> -->
<!-- <module name="JavadocTagContinuationIndentation"/> -->
<!-- <module name="SummaryJavadoc">
<property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
</module> -->
<!-- <module name="JavadocParagraph"/> -->
<!-- <module name="AtclauseOrder">
<property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
<property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
</module> -->
<!-- <module name="JavadocMethod">
<property name="scope" value="public"/>
<property name="allowMissingParamTags" value="true"/>
<property name="allowMissingThrowsTags" value="true"/>
<property name="allowMissingReturnTag" value="true"/>
<property name="minLineCount" value="2"/>
<property name="allowedAnnotations" value="Override, Test"/>
<property name="allowThrowsTagsForSubclasses" value="true"/>
</module> -->
<module name="MethodName">
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
<message key="name.invalidPattern"
value="Method name ''{0}'' must match pattern ''{1}''."/>
</module>
<!-- <module name="SingleLineJavadoc">
<property name="ignoreInlineTags" value="false"/>
</module> -->
<module name="EmptyCatchBlock">
<property name="exceptionVariableName" value="expected"/>
</module>
<module name="CommentsIndentation"/>
<module name="SuppressionCommentFilter"/>
</module>
</module>

2
script/formatting/config/pycodestyle Normal file
View File

@@ -0,0 +1,2 @@
[pycodestyle]
ignore = E501

378
script/formatting/config/pylintrc Normal file
View File

@@ -0,0 +1,378 @@
[MASTER]
# Specify a configuration file.
#rcfile=
# Python code to execute. Adds the analysis directory to the system path
# variable which is necessary to avoid ImportErrors in our website code.
init-hook='import os, sys; cwd = os.getcwd(); analysis_path = os.path.join(cwd, 'server') if '/server/' not in cwd else cwd[0:cwd.index('/server/') + len('/server/') - 1]; sys.path.insert(0, analysis_path)'
# Add files or directories to the blacklist. They should be base names, not
# paths.
ignore=CVS
# Pickle collected data for later comparisons.
persistent=yes
# List of plugins (as comma separated values of python modules names) to load,
# usually to register additional checkers.
load-plugins=
# Use multiple processes to speed up Pylint.
jobs=1
# Allow loading of arbitrary C extensions. Extensions are imported into the
# active Python interpreter and may run arbitrary code.
unsafe-load-any-extension=no
# A comma-separated list of package or module names from where C extensions may
# be loaded. Extensions are loading into the active Python interpreter and may
# run arbitrary code
extension-pkg-whitelist=
# Allow optimization of some AST trees. This will activate a peephole AST
# optimizer, which will apply various small optimizations. For instance, it can
# be used to obtain the result of joining multiple strings with the addition
# operator. Joining a lot of strings can lead to a maximum recursion error in
# Pylint and this flag can prevent that. It has one side effect, the resulting
# AST will be different than the one from reality.
optimize-ast=no
[MESSAGES CONTROL]
# Only show warnings with the listed confidence levels. Leave empty to show
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
confidence=HIGH,INFERENCE_FAILURE,UNDEFINED
# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
# multiple time. See also the "--disable" option for examples.
#enable=
# Disable the message, report, category or checker with the given id(s). You
# can either give multiple identifiers separated by comma (,) or put this
# option multiple times (only on the command line, not in the configuration
# file where it should appear only once).You can also use "--disable=all" to
# disable everything first and then reenable specific checks. For example, if
# you want to run only the similarities checker, you can use "--disable=all
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use"--disable=all --enable=classes
# --disable=W"
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,missing-docstring,too-few-public-methods,too-many-arguments,too-many-locals,too-many-instance-attributes,too-many-statements,locally-disabled,superfluous-parens,too-many-branches,not-callable,too-many-nested-blocks,fixme,redefined-variable-type,no-member,locally-enabled,too-many-public-methods
[REPORTS]
# Set the output format. Available formats are text, parseable, colorized, msvs
# (visual studio) and html. You can also give a reporter class, eg
# mypackage.mymodule.MyReporterClass.
output-format=text
# Put messages in a separate file for each module / package specified on the
# command line instead of printing them on stdout. Reports (if any) will be
# written in a file name "pylint_global.[txt|html]".
files-output=no
# Tells whether to display a full report or only the messages
reports=no
# Python expression which should return a note less than 10 (10 is the highest
# note). You have access to the variables errors warning, statement which
# respectively contain the number of errors / warnings messages and the total
# number of statements analyzed. This is used by the global evaluation report
# (RP0004).
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
# Template used to display messages. This is a python new-style format string
# used to format the message information. See doc for all details
#msg-template=
[SIMILARITIES]
# Minimum lines number of a similarity.
min-similarity-lines=4
# Ignore comments when computing similarities.
ignore-comments=yes
# Ignore docstrings when computing similarities.
ignore-docstrings=yes
# Ignore imports when computing similarities.
ignore-imports=no
[TYPECHECK]
# Tells whether missing members accessed in mixin class should be ignored. A
# mixin class is detected if its name ends with "mixin" (case insensitive).
ignore-mixin-members=yes
# List of module names for which member attributes should not be checked
# (useful for modules/projects where namespaces are manipulated during runtime
# and thus existing member attributes cannot be deduced by static analysis. It
# supports qualified module names, as well as Unix pattern matching.
ignored-modules=
# List of classes names for which member attributes should not be checked
# (useful for classes with attributes dynamically set). This supports can work
# with qualified names.
ignored-classes=
# List of members which are set dynamically and missed by pylint inference
# system, and so shouldn't trigger E1101 when accessed. Python regular
# expressions are accepted.
generated-members=
[LOGGING]
# Logging modules to check that the string format arguments are in logging
# function parameter format
logging-modules=logging
[SPELLING]
# Spelling dictionary name. Available dictionaries: none. To make it working
# install python-enchant package.
spelling-dict=
# List of comma separated words that should not be checked.
spelling-ignore-words=
# A path to a file that contains private dictionary; one word per line.
spelling-private-dict-file=
# Tells whether to store unknown words to indicated private dictionary in
# --spelling-private-dict-file option instead of raising a message.
spelling-store-unknown-words=no
[VARIABLES]
# Tells whether we should check for unused import in __init__ files.
init-import=no
# A regular expression matching the name of dummy variables (i.e. expectedly
# not used).
dummy-variables-rgx=_$|dummy
# List of additional names supposed to be defined in builtins. Remember that
# you should avoid to define new builtins when possible.
additional-builtins=
# List of strings which can identify a callback function by name. A callback
# name must start or end with one of those strings.
callbacks=cb_,_cb
[FORMAT]
# Maximum number of characters on a single line.
max-line-length=100
# Regexp for a line that is allowed to be longer than the limit.
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
# Allow the body of an if to be on the same line as the test if there is no
# else.
single-line-if-stmt=no
# List of optional constructs for which whitespace checking is disabled. `dict-
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
# `empty-line` allows space-only lines.
no-space-check=trailing-comma,dict-separator
# Maximum number of lines in a module
max-module-lines=1000
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
# tab).
indent-string=' '
# Number of spaces of indent required inside a hanging or continued line.
indent-after-paren=4
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
expected-line-ending-format=
[MISCELLANEOUS]
# List of note tags to take in consideration, separated by a comma.
notes=FIXME,XXX,TODO
[BASIC]
# List of builtins function names that should not be used, separated by a comma
bad-functions=map,filter,input
# Good variable names which should always be accepted, separated by a comma
good-names=ex,Run,_,mu,y
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata
# Colon-delimited sets of names that determine each other's naming style when
# the name regexes allow several styles.
name-group=
# Include a hint for the correct naming format with invalid-name
include-naming-hint=no
# Regular expression matching correct function names
function-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for function names
function-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct variable names
variable-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$
# Naming hint for variable names
variable-name-hint=[a-z_][a-z0-9_]{0,30}$
# Regular expression matching correct constant names
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Naming hint for constant names
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
# Regular expression matching correct attribute names
attr-rgx=(([a-z_][a-z0-9_]{1,30})|([XK][a-z0-9_]{0,30}))$
# Naming hint for attribute names
attr-name-hint=[a-z_][a-z0-9_]{1,30}$
# Regular expression matching correct argument names
argument-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$
# Naming hint for argument names
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression matching correct class attribute names
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Naming hint for class attribute names
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
# Regular expression matching correct inline iteration names
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
# Naming hint for inline iteration names
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
# Regular expression matching correct class names
class-rgx=[A-Z_][a-zA-Z0-9]+$
# Naming hint for class names
class-name-hint=[A-Z_][a-zA-Z0-9]+$
# Regular expression matching correct module names
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Naming hint for module names
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
# Regular expression matching correct method names
method-rgx=[a-z_][a-z0-9_]{2,30}$
# Naming hint for method names
method-name-hint=[a-z_][a-z0-9_]{2,30}$
# Regular expression which should only match function or class names that do
# not require a docstring.
no-docstring-rgx=^_
# Minimum line length for functions/classes that require docstrings, shorter
# ones are exempt.
docstring-min-length=-1
[ELIF]
# Maximum number of nested blocks for function / method body
max-nested-blocks=5
[CLASSES]
# List of method names used to declare (i.e. assign) instance attributes.
defining-attr-methods=__init__,__new__,setUp
# List of valid names for the first argument in a class method.
valid-classmethod-first-arg=cls
# List of valid names for the first argument in a metaclass class method.
valid-metaclass-classmethod-first-arg=mcs
# List of member names, which should be excluded from the protected access
# warning.
exclude-protected=_asdict,_fields,_replace,_source,_make
[IMPORTS]
# Deprecated modules which should not be used, separated by a comma
deprecated-modules=regsub,TERMIOS,Bastion,rexec
# Create a graph of every (i.e. internal and external) dependencies in the
# given file (report RP0402 must not be disabled)
import-graph=
# Create a graph of external dependencies in the given file (report RP0402 must
# not be disabled)
ext-import-graph=
# Create a graph of internal dependencies in the given file (report RP0402 must
# not be disabled)
int-import-graph=
[DESIGN]
# Maximum number of arguments for function / method
max-args=5
# Argument names that match this expression will be ignored. Default to name
# with leading underscore
ignored-argument-names=_.*
# Maximum number of locals for function / method body
max-locals=15
# Maximum number of return / yield for function / method body
max-returns=6
# Maximum number of branch for function / method body
max-branches=12
# Maximum number of statements in function / method body
max-statements=50
# Maximum number of parents for a class (see R0901).
max-parents=7
# Maximum number of attributes for a class (see R0902).
max-attributes=7
# Minimum number of public methods for a class (see R0903).
min-public-methods=2
# Maximum number of public methods for a class (see R0904).
max-public-methods=20
# Maximum number of boolean expressions in a if statement
max-bool-expr=5
[EXCEPTIONS]
# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception

198
script/formatting/formatter.py Normal file
View File

@@ -0,0 +1,198 @@
#
# OtterTune - formatter.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
import argparse
import functools
import logging
import os
import re
import subprocess
import sys
import autopep8
EXIT_SUCCESS = 0
EXIT_FAILURE = -1
# ==============================================
# LOGGING CONFIGURATION
# ==============================================
LOG = logging.getLogger(__name__)
LOG_HANDLER = logging.StreamHandler()
LOG_FORMATTER = logging.Formatter(
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
datefmt='%H:%M:%S'
)
LOG_HANDLER.setFormatter(LOG_FORMATTER)
LOG.addHandler(LOG_HANDLER)
LOG.setLevel(logging.INFO)
# ==============================================
# CONFIGURATION
# ==============================================
# NOTE: the absolute path to ottertune directory is calculated from current
# directory structure: ottertune/server/website/scripts/validators/<this_file>
# OTTERTUNE_DIR needs to be redefined if the directory structure is changed.
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
OTTERTUNE_DIR = os.path.abspath(functools.reduce(os.path.join,
[CODE_SOURCE_DIR,
os.path.pardir,
os.path.pardir]))
JAVA_JAR_PATH = os.path.join(
OTTERTUNE_DIR, 'controller/build/libs/google-java-format-1.5-all-deps.jar')
# ==============================================
# FILE HEADER FORMATS
# ==============================================
PYTHON_HEADER_FORMAT = (
"#\n"
"# OtterTune - {filename}\n"
"#\n"
"# Copyright (c) 2017-18, Carnegie Mellon University Database Group\n"
"#\n"
).format
# Regex for updating old headers
PYTHON_HEADER_REGEX = re.compile(r'#\n#.*\n#\n# Copyright.*\n#\n')
JAVA_HEADER_FORMAT = (
"/*\n"
" * OtterTune - {filename}\n"
" *\n"
" * Copyright (c) 2017-18, Carnegie Mellon University Database Group\n"
" */\n\n"
).format
JAVA_HEADER_REGEX = re.compile(r'/\*\n \*.*\n \*\n \* Copyright.*\n \*/\n\n')
# ==============================================
# UTILITY FUNCTION DEFINITIONS
# ==============================================
def format_file(file_path, update_header, format_code):
if file_path.endswith(".py"):
format_python_file(file_path, update_header, format_code)
elif file_path.endswith(".java"):
format_java_file(file_path, update_header, format_code)
def update_file_header(file_contents, file_name, header_format, header_regex):
new_header = header_format(filename=os.path.basename(file_name))
header_match = header_regex.search(file_contents)
if header_match:
# Replace the old header with the new one
old_header = header_match.group()
file_contents = file_contents.replace(old_header, new_header)
else:
# Add new header
file_contents = new_header + file_contents
return file_contents
def format_java_file(file_path, update_header, format_code):
if not file_path.endswith(".java"):
return
if update_header:
with open(file_path, 'r') as f:
file_contents = f.read()
file_contents = update_file_header(file_contents,
os.path.basename(file_path),
JAVA_HEADER_FORMAT,
JAVA_HEADER_REGEX)
with open(file_path, 'w') as f:
f.write(file_contents)
if format_code:
if not os.path.exists(JAVA_JAR_PATH):
controller_dir = os.path.join(OTTERTUNE_DIR, 'controller')
subprocess.check_output(["gradle", "downloadJars"], cwd=controller_dir)
subprocess.check_output(["java", "-jar", JAVA_JAR_PATH, "-r", file_path])
def format_python_file(file_path, update_header, format_code):
if not file_path.endswith(".py"):
return
with open(file_path, 'r') as f:
file_contents = f.read()
if update_header:
file_contents = update_file_header(file_contents,
os.path.basename(file_path),
PYTHON_HEADER_FORMAT,
PYTHON_HEADER_REGEX)
if format_code:
# Use the autopep8 module to format the source code. autopep8 uses
# pycodestyle to detect the style errors it should fix and thus it
# should fix all (or most) of them, however, it does not use pylint
# so it may not fix all of its reported errors.
options = {"max_line_length": 100}
file_contents = autopep8.fix_code(file_contents, options=options)
with open(file_path, 'w') as f:
f.write(file_contents)
# Format all the files in the dir passed as argument
def format_dir(dir_path, update_header, format_code):
for subdir, _, files in os.walk(dir_path): # pylint: disable=not-an-iterable
for file_path in files:
file_path = subdir + os.path.sep + file_path
format_file(file_path, update_header, format_code)
def main():
parser = argparse.ArgumentParser(description='Formats python source files in place')
parser.add_argument('--no-update-header', action='store_true',
help='Do not update the source file headers')
parser.add_argument('--no-format-code', action='store_true',
help='Do not format the source files use autopep8')
parser.add_argument('--staged-files', action='store_true',
help='Apply the selected action(s) to all staged files (git)')
parser.add_argument('paths', metavar='PATH', type=str, nargs='*',
help='Files or directories to (recursively) apply the actions to')
args = parser.parse_args()
if args.no_update_header and args.no_format_code:
LOG.info("No actions to perform (both --no-update-header and "
"--no-format-code given). Exiting...")
sys.exit(EXIT_FAILURE)
elif args.staged_files:
targets = [os.path.abspath(os.path.join(OTTERTUNE_DIR, f))
for f in subprocess.check_output(["git", "diff",
"--name-only", "HEAD",
"--cached",
"--diff-filter=d"]).split()]
if not targets:
LOG.error("No staged files or not calling from a repository. Exiting...")
sys.exit(EXIT_FAILURE)
elif not args.paths:
LOG.error("No files or directories given. Exiting...")
sys.exit(EXIT_FAILURE)
else:
targets = args.paths
for x in targets:
if os.path.isfile(x):
LOG.info("Scanning file: " + x)
format_file(x, not args.no_update_header, not args.no_format_code)
elif os.path.isdir(x):
LOG.info("Scanning directory: " + x)
format_dir(x, not args.no_update_header, not args.no_format_code)
else:
LOG.error("%s isn't a file or directory", x)
sys.exit(EXIT_FAILURE)
if __name__ == '__main__':
main()

79
script/git-hooks/pre-commit Executable file
View File

@@ -0,0 +1,79 @@
#!/bin/sh
# Source validation pre-commit hook
#
# Adapted from the source validation pre-commit hook used in Peloton.
# (see https://github.com/cmu-db/peloton/blob/master/script/git-hooks/pre-commit)
#
# This script collects all modified files and runs it through our source code
# validation script. The validation script returns 0 on success and 1 on any
# failure. This script can also run the server and controller tests by
# uncommenting lines 26-28 and 31-33, respectively.
#
# To enable, symlink this file to '.git/hooks/pre-commit' like so:
# cd $OTTERTUNE_DIR/.git/hooks
# ln -s ../../script/git-hooks/pre-commit ./pre-commit
FILES=$(git diff --name-only HEAD --cached --diff-filter=d | grep '\.\(py\)$')
SERVER_TESTS_RESULT=0
CONTROLLER_TESTS_RESULT=0
VALIDATOR_RESULT=0
if [ -n "$FILES" ]; then
# Uncomment to run the server tests
# cd server/website && python manage.py test -v 2
# SERVER_TESTS_RESULT=$?
# cd ../..
# Uncomment to run the controller tests
# cd controller && gradle build -q
# CONTROLLER_TESTS_RESULT=$?
# cd ..
# Run source code validator
python script/validators/source_validator.py $FILES
VALIDATOR_RESULT=$?
if [ "$VALIDATOR_RESULT" -ne 0 ] || [ "$SERVER_TESTS_RESULT" -ne 0 ] || [ "$CONTROLLER_TESTS_RESULT" -ne 0 ]; then
echo " +------------------------------------------------------------+"
echo " | |"
echo " | OTTERTUNE PRE-COMMIT HOOK |"
echo " | |"
echo " +------------------------------------------------------------+"
echo ""
if [ "$SERVER_TESTS_RESULT" -ne 0 ]; then
echo " FAILED server tests!"
echo ""
fi
if [ "$CONTROLLER_TESTS_RESULT" -ne 0 ]; then
echo " FAILED controller tests!"
echo ""
fi
if [ "$VALIDATOR_RESULT" -ne 0 ]; then
echo " FAILED source validation!"
echo ""
echo " Use the formatting script to help format all changed files:"
echo " (ottertune/script/formatting/formatter.py)"
echo ""
echo " \"python formatter.py --staged-files\""
echo ""
fi
echo " To temporarily bypass the pre-commit hook, use:"
echo ""
echo " \"git commit --no-verify\""
echo
echo " Be aware that changed files have to be staged again!"
exit 1
fi
fi
exit 0

54
script/query_and_get.py Normal file
View File

@@ -0,0 +1,54 @@
#
# OtterTune - query_and_get.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
'''
Created on Feb 11, 2018
@author: taodai
'''
import sys
import time
import logging
import json
import urllib.request
# Logging
LOG = logging.getLogger(__name__)
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)
# take 3 arguments, save result to next_config in working directory
# base_url: for instance, https://0.0.0.0:8000/
# upload_code: upload code...
# query_interval: time (in second) between queries
def main():
base_url = sys.argv[1].strip('/')
upload_code = sys.argv[2]
query_interval = int(sys.argv[3])
request = base_url + '/query_and_get/' + upload_code
timer = 0
start = time.time()
while True:
response = urllib.request.urlopen(request).read().decode()
if 'Fail' in response:
LOG.info('Tuning failed\n')
break
elif response == 'null' or 'not ready' in response:
time.sleep(query_interval)
timer += query_interval
LOG.info('%s s\n', str(timer))
else:
next_conf_f = open('next_config', 'w')
next_conf_f.write(json.loads(response))
next_conf_f.close()
break
elapsed_time = time.time() - start
LOG.info('Elapsed time: %s\n', str(elapsed_time))
if __name__ == "__main__":
main()

437
script/validators/source_validator.py Normal file
View File

@@ -0,0 +1,437 @@
#!/usr/bin/env python
# encoding: utf-8
#
# OtterTune - source_validator.py
#
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
#
# ==============================================
# SOURCE VALIDATOR
# ==============================================
#
# Adapted from the source validator used by Peloton.
# (see https://github.com/cmu-db/peloton/blob/master/script/validators/source_validator.py)
import argparse
import logging
import imp
import os
import re
import subprocess
import sys
import json
import functools
from collections import namedtuple
from fabric.api import lcd, local, settings, quiet
EXIT_SUCCESS = 0
EXIT_FAILURE = -1
# ==============================================
# CONFIGURATION
# ==============================================
# Logging
LOG = logging.getLogger(__name__)
LOG.addHandler(logging.StreamHandler())
LOG.setLevel(logging.INFO)
# NOTE: the absolute path to ottertune directory is calculated from current
# directory structure: ottertune/server/website/scripts/validators/<this_file>
# OTTERTUNE_DIR needs to be redefined if the directory structure is changed.
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
OTTERTUNE_DIR = os.path.abspath(functools.reduce(os.path.join,
[CODE_SOURCE_DIR,
os.path.pardir,
os.path.pardir]))
# Other directory paths used are relative to OTTERTUNE_DIR
DEFAULT_DIRS = [
OTTERTUNE_DIR
]
# Directories that should NOT be checked
EXCLUDE_DIRECTORIES = [
# Django-generated directories
os.path.join(OTTERTUNE_DIR, "server/website/website/migrations"),
# Source code files from json.org
os.path.join(OTTERTUNE_DIR, "client/controller/src/main/java/com/controller/util/json"),
# Django settings
os.path.join(OTTERTUNE_DIR, 'server/website/website/settings'),
# Docker files
os.path.join(OTTERTUNE_DIR, 'docker'),
]
# Files that should NOT be checked
EXCLUDE_FILES = [
# Django-generated files
os.path.join(OTTERTUNE_DIR, 'server/website/manage.py'),
# Docker files
os.path.join(OTTERTUNE_DIR, 'server/website/createadmin.py'),
]
CHECKSTYLE_JAR_PATH = os.path.join(OTTERTUNE_DIR,
"client/controller/build/libs/checkstyle-8.8-all.jar")
# Regex patterns
PYCODESTYLE_COMMENT_PATTERN = re.compile(r'#\s*pycodestyle:\s*disable\s*=\s*[\w\,\s]+$')
PYTHON_ILLEGAL_PATTERNS = [
(re.compile(r'^print[ (]'), "Do not use 'print'. Use the logging module instead.")
]
JAVA_ILLEGAL_PATTERNS = [
(re.compile(r'^System.out.println'), "Do not use println. Use the logging module instead.")
]
PYTHON_HEADER_PATTERN = re.compile(r'#\n#.*\n#\n# Copyright.*\n#\n')
JAVA_HEADER_PATTERN = re.compile(r'/\*\n \*.*\n \*\n \* Copyright.*\n \*/\n\n')
# Stdout format strings
SEPARATOR = 80 * '-'
OUTPUT_FMT = (
'' + SEPARATOR + '\n\n'
'\033[1m' # start bold text
'%s\n'
'FAILED: %s\n\n'
'\033[0m' # end bold text
'%s'
)
VALIDATOR_FMT = '{name}\n{u}\n{out}'.format
MSG_PREFIX_FMT = ' {filename}:{line:3d}: '.format
MSG_SUFFIX_FMT = ' ({symbol})'.format
# ==============================================
# UTILITY FUNCTION DEFINITIONS
# ==============================================
def format_message(filename, line, message, symbol=None):
out_prefix = MSG_PREFIX_FMT(filename=filename, line=line)
out_suffix = '' if symbol is None else MSG_SUFFIX_FMT(symbol=symbol)
# Crop the message details to make the output more readable
max_msg_len = 80 - len(out_prefix) - len(out_suffix)
if len(message) > max_msg_len:
message = message[:max_msg_len - 3] + '...'
output = (out_prefix + message + out_suffix).replace('\n', '')
return output + '\n'
def validate_validator(modules, config_path):
status = True
# Check if required modules are installed
for module in modules:
if module is not None:
try:
imp.find_module(module)
except ImportError:
LOG.error("Cannot find module %s", module)
status = False
# Check that the config file exists if assigned
if config_path is not None and not os.path.isfile(config_path):
LOG.error("Cannot find config file %s", config_path)
status = False
return status
# Validate the file passed as argument
def validate_file(file_path):
if file_path in EXCLUDE_FILES:
return True
if not file_path.endswith(".py") and not file_path.endswith(".java"):
return True
LOG.debug("Validating file: %s", file_path)
status = True
output = []
failed_validators = []
for validator in VALIDATORS:
val_status, val_output = validator.validate_fn(
file_path, validator.config_path)
if not val_status:
status = False
output.append(VALIDATOR_FMT(name=validator.name,
u='-' * len(validator.name),
out=val_output))
failed_validators.append(validator.name)
if not status:
LOG.info(OUTPUT_FMT, file_path, ', '.join(failed_validators), '\n'.join(output))
return status
# Validate all the files in the root_dir passed as argument
def validate_dir(root_dir):
if root_dir in EXCLUDE_DIRECTORIES:
return True
status = True
for root, dirs, files in os.walk(root_dir): # pylint: disable=not-an-iterable
# Remove excluded dirs from list
dirs[:] = [d for d in dirs if os.path.join(root, d) not in EXCLUDE_DIRECTORIES]
for file_path in files:
file_path = os.path.join(root, file_path)
if not validate_file(file_path):
status = False
return status
# ==============================================
# VALIDATOR FUNCTION DEFINITIONS
# ==============================================
def check_pylint(file_path, config_path=None):
if not file_path.endswith(".py"):
return True, None
options = [
'--output-format=json',
'--reports=yes',
]
if config_path is not None:
options.append('--rcfile=' + config_path)
with settings(warn_only=True), quiet():
res = local('pylint {} {}'.format(' '.join(options), file_path), capture=True)
if res.stdout == '':
assert res.return_code == 0, 'return_code={}, expected=0\n{}'.format(
res.return_code, res.stderr)
return True, None
output = []
errors = json.loads(res.stdout)
for entry in errors:
# Remove extra whitespace and hints
msg = entry['message'].replace('^', '').replace('|', '')
msg = re.sub(' +', ' ', msg)
msg = msg.strip()
output.append(format_message(os.path.basename(file_path), entry['line'],
msg, entry['symbol']))
output = ''.join(output)
return res.return_code == 0, output
def check_pycodestyle(file_path, config_path=None):
import pycodestyle
if not file_path.endswith(".py"):
return True, None
# A custom reporter class for pycodestyle that checks for disabled errors
# and formats the style report output.
class CustomReporter(pycodestyle.StandardReport):
def get_file_results(self):
# Iterates through the lines of code that generated lint errors and
# checks if the given error has been disabled for that line via an
# inline comment (e.g., # pycodestyle: disable=E201,E226). Those
# that have been disabled are not treated as errors.
self._deferred_print.sort()
results = []
prev_line_num = -1
prev_line_errs = []
for line_number, _, code, text, _ in self._deferred_print:
if prev_line_num == line_number:
err_codes = prev_line_errs
else:
line = self.lines[line_number - 1]
m = PYCODESTYLE_COMMENT_PATTERN.search(line)
if m and m.group(0):
err_codes = [ec.strip() for ec in m.group(0).split('=')[1].split(',')]
else:
err_codes = []
prev_line_num = line_number
prev_line_errs = err_codes
if code in err_codes:
# Error is disabled in source
continue
results.append(format_message(os.path.basename(file_path),
self.line_offset + line_number,
text, code))
return results, len(results) == 0
# END CustomReporter class
options = {} if config_path is None else {'config_file': config_path}
style = pycodestyle.StyleGuide(quiet=True, **options)
# Set the reporter option to our custom one
style.options.reporter = CustomReporter
style.init_report()
report = style.check_files([file_path])
results, status = report.get_file_results()
output = None if status else ''.join(results)
return status, output
def check_java_checkstyle(file_path, config_path=None):
if not file_path.endswith(".java"):
return True, None
if not os.path.exists(CHECKSTYLE_JAR_PATH):
with lcd(os.path.join(OTTERTUNE_DIR, "client/controller")): # pylint: disable=not-context-manager
local("gradle downloadJars")
options = '' if config_path is None else '-c ' + config_path
with quiet():
res = local("java -jar {} {} {}".format(CHECKSTYLE_JAR_PATH, options, file_path),
capture=True)
lines = res.stdout.split('\n')
assert len(lines) >= 2 and lines[0] == "Starting audit..." and lines[-1] == "Audit done."
if len(lines) == 2:
return True, None
output = []
for line in lines[1:-1]:
parts = line.strip().split(':')
line_number = int(parts[1])
text, code = parts[-1].rsplit('[', 1)
text = text.strip()
code = code[:-1]
output.append(format_message(os.path.basename(file_path), line_number, text, code))
output = ''.join(output)
return False, output
def check_illegal_patterns(file_path, config_path=None): # pylint: disable=unused-argument
if file_path.endswith(".py"):
illegal_patterns = PYTHON_ILLEGAL_PATTERNS
comment = "#"
elif file_path.endswith(".java"):
illegal_patterns = JAVA_ILLEGAL_PATTERNS
comment = "//"
else:
return True, None
line_num = 1
output = []
status = True
with open(file_path, 'r') as f:
for line in f:
line = line.strip()
for pattern_info in illegal_patterns:
if not line.startswith(comment) and pattern_info[0].search(line):
output.append(format_message(filename=os.path.basename(file_path),
line=line_num,
message=pattern_info[1]))
status = False
line_num += 1
output = None if status else ''.join(output)
return status, output
def check_header(file_path, config_file=None): # pylint: disable=unused-argument
if file_path.endswith(".py"):
header_pattern = PYTHON_HEADER_PATTERN
elif file_path.endswith(".java"):
header_pattern = JAVA_HEADER_PATTERN
else:
return True, None
status = True
output = None
with open(file_path, 'r') as f:
file_contents = f.read()
header_match = header_pattern.search(file_contents)
filename = os.path.basename(file_path)
if header_match:
if filename not in header_match.group(0):
status = False
output = format_message(filename=filename, line=2,
message="Incorrect filename in header")
else:
status = False
output = format_message(filename=filename, line=1,
message='Missing header')
return status, output
# ==============================================
# VALIDATORS
# ==============================================
# Struct for storing validator metadata
Validator = namedtuple('Validator', 'name validate_fn modules config_path')
VALIDATORS = [
# Runs pylint on python source
Validator('check_pylint', check_pylint, ['pylint'],
os.path.join(OTTERTUNE_DIR, "script/formatting/config/pylintrc")),
# Runs pycodestyle on python source
Validator('check_pycodestyle', check_pycodestyle, ['pycodestyle'],
os.path.join(OTTERTUNE_DIR, "script/formatting/config/pycodestyle")),
# Runs checkstyle on the java source
Validator("check_java_checkstyle", check_java_checkstyle, [],
os.path.join(OTTERTUNE_DIR, "script/formatting/config/google_checks.xml")),
# Checks that the python/java source files do not use illegal patterns
Validator('check_illegal_patterns', check_illegal_patterns, [], None),
# Checks that the python/java source files have headers
Validator('check_header', check_header, [], None)
]
# ==============================================
# MAIN FUNCTION
# ==============================================
def main():
parser = argparse.ArgumentParser(description="Validate OtterTune's source code")
parser.add_argument('paths', metavar='PATH', type=str, nargs='*',
help='Files or directories to (recursively) validate')
parser.add_argument('--staged-files', action='store_true',
help='Apply the selected action(s) to all staged files (git)')
args = parser.parse_args()
LOG.info('\nRunning source validators:\n%s\n',
'\n'.join(' ' + v.name for v in VALIDATORS))
for validator in VALIDATORS:
if not validate_validator(validator.modules, validator.config_path):
sys.exit(EXIT_FAILURE)
if args.staged_files:
targets = [os.path.abspath(os.path.join(OTTERTUNE_DIR, f))
for f in subprocess.check_output(["git", "diff", "--name-only", "HEAD",
"--cached", "--diff-filter=d"]).split()]
if not targets:
LOG.error("No staged files or not calling from a repository. Exiting...")
sys.exit(EXIT_FAILURE)
elif args.paths:
targets = args.paths
else:
targets = DEFAULT_DIRS
for target in targets:
target = os.path.abspath(target)
if os.path.isfile(target):
LOG.debug("Scanning file: %s\n", target)
status = validate_file(target)
elif os.path.isdir(target):
LOG.debug("Scanning directory: %s\n", target)
status = validate_dir(target)
else:
LOG.error("%s isn't a file or directory", target)
sys.exit(EXIT_FAILURE)
if not status:
LOG.info(SEPARATOR + '\n')
LOG.info("Validation NOT successful\n")
sys.exit(EXIT_FAILURE)
LOG.info("Validation successful\n")
sys.exit(EXIT_SUCCESS)
if __name__ == '__main__':
main()