Initial commit with BSL
This commit is contained in:
Andy Pavlo
commit
3e564ce922
|
|
@ -0,0 +1,6 @@
|
||||||
|
*.swp
|
||||||
|
*~
|
||||||
|
*.pyc
|
||||||
|
*.md
|
||||||
|
.travis.yml
|
||||||
|
LICENSE
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
# Byte-compiled / optimized / DLL files #
|
||||||
|
#########################################
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
*$py.class
|
||||||
|
*.com
|
||||||
|
*.class
|
||||||
|
*.dll
|
||||||
|
*.exe
|
||||||
|
*.o
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# OS generated files #
|
||||||
|
######################
|
||||||
|
.DS_Store
|
||||||
|
.DS_Store?
|
||||||
|
._*
|
||||||
|
.Spotlight-V100
|
||||||
|
.Trashes
|
||||||
|
ehthumbs.db
|
||||||
|
Thumbs.db
|
||||||
|
|
||||||
|
# Distribution / packaging #
|
||||||
|
############################
|
||||||
|
.Python
|
||||||
|
env/
|
||||||
|
build/
|
||||||
|
develop-eggs/
|
||||||
|
dist/
|
||||||
|
downloads/
|
||||||
|
eggs/
|
||||||
|
.eggs/
|
||||||
|
lib/
|
||||||
|
lib64/
|
||||||
|
parts/
|
||||||
|
sdist/
|
||||||
|
var/
|
||||||
|
*.egg-info/
|
||||||
|
.installed.cfg
|
||||||
|
*.egg
|
||||||
|
|
||||||
|
# Packages #
|
||||||
|
############
|
||||||
|
# it's better to unpack these files and commit the raw source
|
||||||
|
# git has its own built in compression methods
|
||||||
|
*.7z
|
||||||
|
*.dmg
|
||||||
|
*.gz
|
||||||
|
*.iso
|
||||||
|
*.jar
|
||||||
|
*.rar
|
||||||
|
*.tar
|
||||||
|
*.zip
|
||||||
|
|
||||||
|
# PyInstaller #
|
||||||
|
###############
|
||||||
|
*.manifest
|
||||||
|
*.spec
|
||||||
|
|
||||||
|
# Installer logs #
|
||||||
|
##################
|
||||||
|
pip-log.txt
|
||||||
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
|
# Unit test / coverage reports #
|
||||||
|
################################
|
||||||
|
htmlcov/
|
||||||
|
.tox/
|
||||||
|
.coverage
|
||||||
|
.coverage.*
|
||||||
|
.cache
|
||||||
|
nosetests.xml
|
||||||
|
coverage.xml
|
||||||
|
*,cover
|
||||||
|
.hypothesis/
|
||||||
|
|
||||||
|
# Env #
|
||||||
|
#######
|
||||||
|
.python-version
|
||||||
|
.env
|
||||||
|
venv/
|
||||||
|
ENV/
|
||||||
|
|
||||||
|
# Eclipse #
|
||||||
|
###########
|
||||||
|
.project
|
||||||
|
.pydevproject
|
||||||
|
.settings
|
||||||
|
.classpath
|
||||||
|
|
||||||
|
# Intellij and PyCharm #
|
||||||
|
.idea
|
||||||
|
*.iml
|
||||||
|
*.iws
|
||||||
|
out/
|
||||||
|
|
||||||
|
# vim #
|
||||||
|
*~
|
||||||
|
*.swp
|
||||||
|
*.swo
|
||||||
|
|
||||||
|
# Text editor configs #
|
||||||
|
#######################
|
||||||
|
.vimrc
|
||||||
|
|
@ -0,0 +1,58 @@
|
||||||
|
dist: bionic
|
||||||
|
sudo: required
|
||||||
|
|
||||||
|
services:
|
||||||
|
- docker
|
||||||
|
|
||||||
|
env:
|
||||||
|
global:
|
||||||
|
- DOCKER_COMPOSE_VERSION=1.24.1
|
||||||
|
matrix:
|
||||||
|
- DOCKER_OS=ubuntu-18.04
|
||||||
|
- DOCKER_OS=centos-7
|
||||||
|
|
||||||
|
before_install:
|
||||||
|
# Install latest versions of docker and docker-compose
|
||||||
|
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
|
||||||
|
- sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
|
||||||
|
- sudo apt-get update
|
||||||
|
- sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
|
||||||
|
- sudo rm /usr/local/bin/docker-compose
|
||||||
|
- curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
|
||||||
|
- chmod +x docker-compose
|
||||||
|
- sudo mv docker-compose /usr/local/bin
|
||||||
|
- docker --version
|
||||||
|
- docker-compose --version
|
||||||
|
|
||||||
|
before_script:
|
||||||
|
# Set correct dockerfile
|
||||||
|
- cd $TRAVIS_BUILD_DIR/docker
|
||||||
|
- sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.yml
|
||||||
|
- sed -i "s|Dockerfile\.base-.*|Dockerfile\.base-$DOCKER_OS|" docker-compose.test.yml
|
||||||
|
- sudo service mysql stop || true
|
||||||
|
|
||||||
|
script:
|
||||||
|
- cd $TRAVIS_BUILD_DIR/docker
|
||||||
|
# Build master images and run the webserver
|
||||||
|
- docker-compose build
|
||||||
|
- docker-compose up -d
|
||||||
|
- docker logs ottertune
|
||||||
|
- docker-compose rm -f -s -v
|
||||||
|
# Build test images and run tests
|
||||||
|
- ci_env=`bash <(curl -s https://codecov.io/env)`
|
||||||
|
- docker-compose -f docker-compose.test.yml build
|
||||||
|
- docker-compose -f docker-compose.test.yml up -d
|
||||||
|
- docker-compose -f docker-compose.test.yml run --workdir="/app/client/controller" --rm test gradle build
|
||||||
|
- docker-compose -f docker-compose.test.yml run --workdir="/app/server" --rm $ci_env test bash -c "coverage run --omit=\"*/tests/*\" -m unittest discover -s analysis/tests -v && (codecov -F analysis || (sleep 5 && codecov -F analysis) || (sleep 5 && codecov -F analysis))"
|
||||||
|
- docker-compose -f docker-compose.test.yml run --workdir="/app/server/website" --rm $ci_env test bash -c "./wait-for-it.sh && coverage run --source=website manage.py test --noinput -v 2 && (codecov -F website || (sleep 5 && codecov -F website) || (sleep 5 && codecov -F website))"
|
||||||
|
# Only run source validation once (on ubuntu 18.04)
|
||||||
|
- if [ $DOCKER_OS == ubuntu-18.04 ]; then
|
||||||
|
docker-compose -f docker-compose.test.yml run --workdir="/app" --rm test python3 script/validators/source_validator.py;
|
||||||
|
fi
|
||||||
|
|
||||||
|
after_script:
|
||||||
|
# Cleanup docker containers, images, and volumes
|
||||||
|
- docker-compose rm -f -s -v
|
||||||
|
- docker system prune -a -f
|
||||||
|
- docker volume prune -f
|
||||||
|
|
||||||
|
|
@ -0,0 +1,100 @@
|
||||||
|
Business Source License 1.1
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
|
||||||
|
Licensor: Carnegie Mellon University
|
||||||
|
Licensed Work: OtterTune
|
||||||
|
The Licensed Work is (c) 2019 Carnegie Mellon University
|
||||||
|
Additional Use Grant: You may make use of the Licensed Work, provided that
|
||||||
|
you may not use the Licensed Work for a Database
|
||||||
|
Service.
|
||||||
|
|
||||||
|
A “Database Service” is a commercial offering that
|
||||||
|
allows third parties (other than your employees and
|
||||||
|
contractors) to access the functionality of the
|
||||||
|
Licensed Work by creating tables whose schemas are
|
||||||
|
controlled by such third parties.
|
||||||
|
|
||||||
|
Change Date: 2024-01-18
|
||||||
|
|
||||||
|
Change License: Apache License, Version 2.0
|
||||||
|
|
||||||
|
Notice
|
||||||
|
|
||||||
|
The Business Source License (this document, or the “License”) is not an Open
|
||||||
|
Source license. However, the Licensed Work will eventually be made available
|
||||||
|
under an Open Source License, as stated in this License.
|
||||||
|
|
||||||
|
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
|
||||||
|
“Business Source License” is a trademark of MariaDB Corporation Ab.
|
||||||
|
|
||||||
|
-----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
Business Source License 1.1
|
||||||
|
|
||||||
|
Terms
|
||||||
|
|
||||||
|
The Licensor hereby grants you the right to copy, modify, create derivative
|
||||||
|
works, redistribute, and make non-production use of the Licensed Work. The
|
||||||
|
Licensor may make an Additional Use Grant, above, permitting limited
|
||||||
|
production use.
|
||||||
|
|
||||||
|
Effective on the Change Date, or the fourth anniversary of the first publicly
|
||||||
|
available distribution of a specific version of the Licensed Work under this
|
||||||
|
License, whichever comes first, the Licensor hereby grants you rights under
|
||||||
|
the terms of the Change License, and the rights granted in the paragraph
|
||||||
|
above terminate.
|
||||||
|
|
||||||
|
If your use of the Licensed Work does not comply with the requirements
|
||||||
|
currently in effect as described in this License, you must purchase a
|
||||||
|
commercial license from the Licensor, its affiliated entities, or authorized
|
||||||
|
resellers, or you must refrain from using the Licensed Work.
|
||||||
|
|
||||||
|
All copies of the original and modified Licensed Work, and derivative works
|
||||||
|
of the Licensed Work, are subject to this License. This License applies
|
||||||
|
separately for each version of the Licensed Work and the Change Date may vary
|
||||||
|
for each version of the Licensed Work released by Licensor.
|
||||||
|
|
||||||
|
You must conspicuously display this License on each original or modified copy
|
||||||
|
of the Licensed Work. If you receive the Licensed Work in original or
|
||||||
|
modified form from a third party, the terms and conditions set forth in this
|
||||||
|
License apply to your use of that work.
|
||||||
|
|
||||||
|
Any use of the Licensed Work in violation of this License will automatically
|
||||||
|
terminate your rights under this License for the current and all other
|
||||||
|
versions of the Licensed Work.
|
||||||
|
|
||||||
|
This License does not grant you any right in any trademark or logo of
|
||||||
|
Licensor or its affiliates (provided that you may use a trademark or logo of
|
||||||
|
Licensor as expressly required by this License).
|
||||||
|
|
||||||
|
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
|
||||||
|
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
|
||||||
|
TITLE.
|
||||||
|
|
||||||
|
MariaDB hereby grants you permission to use this License’s text to license
|
||||||
|
your works, and to refer to it using the trademark “Business Source License”,
|
||||||
|
as long as you comply with the Covenants of Licensor below.
|
||||||
|
|
||||||
|
Covenants of Licensor
|
||||||
|
|
||||||
|
In consideration of the right to use this License’s text and the “Business
|
||||||
|
Source License” name and trademark, Licensor covenants to MariaDB, and to all
|
||||||
|
other recipients of the licensed work to be provided by Licensor:
|
||||||
|
|
||||||
|
1. To specify as the Change License the GPL Version 2.0 or any later version,
|
||||||
|
or a license that is compatible with GPL Version 2.0 or a later version,
|
||||||
|
where “compatible” means that software provided under the Change License can
|
||||||
|
be included in a program with software provided under GPL Version 2.0 or a
|
||||||
|
later version. Licensor may specify additional Change Licenses without
|
||||||
|
limitation.
|
||||||
|
|
||||||
|
2. To either: (a) specify an additional grant of rights to use that does not
|
||||||
|
impose any additional restriction on the right granted in this License, as
|
||||||
|
the Additional Use Grant; or (b) insert the text “None”.
|
||||||
|
|
||||||
|
3. To specify a Change Date.
|
||||||
|
|
||||||
|
4. Not to modify this License in any other way.
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
# OtterTune
|
||||||
|
|
||||||
|
[](https://travis-ci.org/cmu-db/ottertune)
|
||||||
|
[](https://codecov.io/github/cmu-db/ottertune)
|
||||||
|
|
||||||
|
OtterTune is a new tool developed by students and researchers in the [Carnegie Mellon Database Group](http://db.cs.cmu.edu/projects/autotune/) that can automatically find good settings for a database management system's configuration knobs. The goal is to make it easier for anyone to deploy a DBMS without any expertise in database administration. To tune new DBMS deployments, OtterTune reuses training data gathered from previous tuning sessions. Because OtterTune does not need to generate an initial dataset for training its ML models, tuning time is drastically reduced.
|
||||||
|
|
||||||
|
For more information, see our [paper](http://db.cs.cmu.edu/papers/2017/p1009-van-aken.pdf).
|
||||||
|
|
||||||
|
```
|
||||||
|
@inproceedings{vanaken17,
|
||||||
|
author = {Van Aken, Dana and Pavlo, Andrew and Gordon, Geoffrey J. and Zhang, Bohan},
|
||||||
|
title = {Automatic Database Management System Tuning Through Large-scale Machine Learning},
|
||||||
|
booktitle = {Proceedings of the 2017 ACM International Conference on Management of Data},
|
||||||
|
series = {SIGMOD '17},
|
||||||
|
year = {2017},
|
||||||
|
pages = {1009--1024},
|
||||||
|
numpages = {16},
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Contributors
|
||||||
|
|
||||||
|
See the [people page](https://github.com/cmu-db/ottertune/graphs/contributors) for the full list of contributors.
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
# Mac OS X hidden file
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# workspace configuration files
|
||||||
|
.settings/
|
||||||
|
.metadata/
|
||||||
|
|
||||||
|
# Gradle
|
||||||
|
.gradle/
|
||||||
|
/gradlew.bat
|
||||||
|
/gradlew
|
||||||
|
/settings.gradle
|
||||||
|
|
||||||
|
# Intellij
|
||||||
|
.idea
|
||||||
|
/dbcollector.iml
|
||||||
|
|
||||||
|
# generated files
|
||||||
|
bin/
|
||||||
|
build/
|
||||||
|
out/
|
||||||
|
output/
|
||||||
|
|
||||||
|
# lib
|
||||||
|
lib/
|
||||||
|
|
||||||
|
# log file
|
||||||
|
*.log
|
||||||
|
|
@ -0,0 +1,12 @@
|
||||||
|
## OtterTune Controller
|
||||||
|
The controller is responsible for collecting database metrics and knobs information during an experiment.</br>
|
||||||
|
#### Usage:
|
||||||
|
To build the project, run `gradle build`.</br>
|
||||||
|
To run the controller, you need to provide a configuration file and provide command line arguments (command line arguments are optional). Then run `gradle run`.
|
||||||
|
|
||||||
|
* Command line arguments:
|
||||||
|
* time (flag : `-t`) </br>
|
||||||
|
The duration of the experiment in `seconds`. The default time is set to 300 seconds.
|
||||||
|
* configuration file path (flag : `-c`) </br>
|
||||||
|
The path of the input configuration file (required). Sample config files are under the directory `config`.
|
||||||
|
|
||||||
|
|
@ -0,0 +1,90 @@
|
||||||
|
plugins {
|
||||||
|
id "de.undercouch.download" version "3.3.0"
|
||||||
|
id "com.github.spotbugs" version "2.0.0"
|
||||||
|
}
|
||||||
|
|
||||||
|
apply plugin: 'java'
|
||||||
|
apply plugin: 'maven'
|
||||||
|
apply plugin: 'application'
|
||||||
|
apply plugin: 'project-report'
|
||||||
|
|
||||||
|
mainClassName = "com.controller.Main"
|
||||||
|
|
||||||
|
test.testLogging { exceptionFormat "full"; events "failed", "passed", "skipped" }
|
||||||
|
|
||||||
|
repositories {
|
||||||
|
mavenCentral()
|
||||||
|
}
|
||||||
|
|
||||||
|
dependencies {
|
||||||
|
testCompile group: 'junit', name: 'junit', version: '4.11'
|
||||||
|
runtime fileTree(dir: 'lib', include: '*.jar')
|
||||||
|
compile group: 'net.sourceforge.collections', name: 'collections-generic', version: '4.01'
|
||||||
|
compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
|
||||||
|
compile group: 'log4j', name: 'log4j', version: '1.2.17'
|
||||||
|
compile group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1'
|
||||||
|
compile group: 'org.apache.httpcomponents', name: 'httpclient', version: '4.5.3'
|
||||||
|
compile group: 'org.apache.httpcomponents', name: 'httpmime', version: '4.5.3'
|
||||||
|
|
||||||
|
// https://mvnrepository.com/artifact/com.github.fge/json-schema-validator
|
||||||
|
compile group: 'com.github.fge', name: 'json-schema-validator', version: '2.2.6'
|
||||||
|
// https://mvnrepository.com/artifact/com.github.fge/jackson-coreutils
|
||||||
|
compile group: 'com.github.fge', name: 'jackson-coreutils', version: '1.8'
|
||||||
|
// https://mvnrepository.com/artifact/com.github.fge/json-schema-core
|
||||||
|
compile group: 'com.github.fge', name: 'json-schema-core', version: '1.2.5'
|
||||||
|
|
||||||
|
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-annotations
|
||||||
|
compile group: 'com.fasterxml.jackson.core', name: 'jackson-annotations', version: '2.9.0'
|
||||||
|
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core
|
||||||
|
compile group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.8.0'
|
||||||
|
// https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind
|
||||||
|
compile group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.8.0'
|
||||||
|
compile(group: "com.github.java-json-tools", name: "json-schema-validator", version: "2.2.8");
|
||||||
|
compile group: 'org.apache.logging.log4j', name: 'log4j-api', version: '2.10.0'
|
||||||
|
compile group: 'org.apache.logging.log4j', name: 'log4j-core', version: '2.10.0'
|
||||||
|
// https://mvnrepository.com/artifact/commons-cli/commons-cli
|
||||||
|
compile group: 'commons-cli', name: 'commons-cli', version: '1.2'
|
||||||
|
|
||||||
|
// https://mvnrepository.com/artifact/mysql/mysql-connector-java
|
||||||
|
compile group: 'mysql', name: 'mysql-connector-java', version: '5.1.6'
|
||||||
|
|
||||||
|
// https://mvnrepository.com/artifact/org.postgresql/postgresql
|
||||||
|
compile group: 'org.postgresql', name: 'postgresql', version: '9.4-1201-jdbc41'
|
||||||
|
|
||||||
|
// This lib has to be manually downloaded from Oracle
|
||||||
|
dependencies {compile files('lib/ojdbc8.jar')}
|
||||||
|
}
|
||||||
|
|
||||||
|
run {
|
||||||
|
if (project.hasProperty("appArgs")) {
|
||||||
|
args(appArgs.split(' '))
|
||||||
|
}
|
||||||
|
// Below is another way to add args but this causes gradle to
|
||||||
|
// always invoke the run application task
|
||||||
|
// args = ["-c","$config","-t","$time","-d","$dir"]
|
||||||
|
}
|
||||||
|
|
||||||
|
spotbugs {
|
||||||
|
toolVersion = '4.0.0-beta3'
|
||||||
|
ignoreFailures = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Note (07-29-2019): the HTML report for spotbugs is currently broken
|
||||||
|
tasks.withType(com.github.spotbugs.SpotBugsTask) {
|
||||||
|
reports {
|
||||||
|
xml.enabled true
|
||||||
|
html.enabled false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
import de.undercouch.gradle.tasks.download.Download
|
||||||
|
task downloadJars(type: Download) {
|
||||||
|
src ([
|
||||||
|
'https://github.com/google/google-java-format/releases/download/google-java-format-1.5/google-java-format-1.5-all-deps.jar',
|
||||||
|
'https://github.com/checkstyle/checkstyle/releases/download/checkstyle-8.8/checkstyle-8.8-all.jar'
|
||||||
|
])
|
||||||
|
dest libsDir
|
||||||
|
overwrite false
|
||||||
|
}
|
||||||
|
|
||||||
|
build.finalizedBy(downloadJars)
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"database_type" : "mysql",
|
||||||
|
"database_url" : "jdbc:mysql://localhost:3306/mysqldb",
|
||||||
|
"username" : "MY_DATABASE_USERNAME",
|
||||||
|
"password" : "MY_DATABASE_PASSWORD",
|
||||||
|
"upload_code" : "DEPRECATED",
|
||||||
|
"upload_url" : "DEPRECATED",
|
||||||
|
"workload_name" : "workload_name"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"database_type" : "oracle",
|
||||||
|
"database_url" : "jdbc:oracle:thin:@localhost:1521:orcldb",
|
||||||
|
"username" : "sys as sysdba",
|
||||||
|
"password" : "oracle",
|
||||||
|
"upload_code" : "DEPRECATED",
|
||||||
|
"upload_url" : "DEPRECATED",
|
||||||
|
"workload_name" : "tpcc"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"database_type" : "postgres",
|
||||||
|
"database_url" : "jdbc:postgresql://localhost:5432/postgres",
|
||||||
|
"username" : "MY_DATABASE_USERNAME",
|
||||||
|
"password" : "MY_DATABASE_PASSWORD",
|
||||||
|
"upload_code" : "DEPRECATED",
|
||||||
|
"upload_url" : "DEPRECATED",
|
||||||
|
"workload_name" : "workload_name"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
{
|
||||||
|
"database_type" : "saphana",
|
||||||
|
"database_url" : "jdbc:sap://localhost:39015",
|
||||||
|
"username" : "MY_DATABASE_USERNAME",
|
||||||
|
"password" : "MY_DATABASE_PASSWORD",
|
||||||
|
"upload_code" : "DEPRECATED",
|
||||||
|
"upload_url" : "DEPRECATED",
|
||||||
|
"workload_name" : "workload_name"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,6 @@
|
||||||
|
#Thu Nov 30 15:45:05 EST 2017
|
||||||
|
distributionBase=GRADLE_USER_HOME
|
||||||
|
distributionPath=wrapper/dists
|
||||||
|
zipStoreBase=GRADLE_USER_HOME
|
||||||
|
zipStorePath=wrapper/dists
|
||||||
|
distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
# Set root logger level to DEBUG and its only appender to A1.
|
||||||
|
log4j.rootLogger=INFO, A1, FILE
|
||||||
|
log4j.rootLogger.layout=org.apache.log4j.PatternLayout
|
||||||
|
|
||||||
|
# A1 is set to be a ConsoleAppender.
|
||||||
|
log4j.appender.A1=org.apache.log4j.ConsoleAppender
|
||||||
|
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} (%F:%L) %-5p - %m%n
|
||||||
|
|
||||||
|
# Redirect log messages to a log file, support file rolling.
|
||||||
|
# Define the file appender
|
||||||
|
log4j.appender.FILE=org.apache.log4j.FileAppender
|
||||||
|
|
||||||
|
# Set the name of the file
|
||||||
|
log4j.appender.FILE.file=controller.log
|
||||||
|
|
||||||
|
# Set the immediate flush to true (default)
|
||||||
|
log4j.appender.FILE.immediateFlush=true
|
||||||
|
|
||||||
|
# Set the threshold to debug mode
|
||||||
|
log4j.appender.FILE.Threshold=debug
|
||||||
|
|
||||||
|
# Set the append to false, overwrite
|
||||||
|
log4j.appender.FILE.append=true
|
||||||
|
|
||||||
|
# Define the layout for file appender
|
||||||
|
log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
|
||||||
|
log4j.appender.A1.layout.ConversionPattern=%d{ABSOLUTE} (%F:%L) %-5p - %m%n
|
||||||
|
|
@ -0,0 +1,517 @@
|
||||||
|
{
|
||||||
|
"global": {"global": {
|
||||||
|
"auto_generate_certs": "ON",
|
||||||
|
"auto_increment_increment": "1",
|
||||||
|
"auto_increment_offset": "1",
|
||||||
|
"autocommit": "ON",
|
||||||
|
"automatic_sp_privileges": "ON",
|
||||||
|
"avoid_temporal_upgrade": "OFF",
|
||||||
|
"back_log": "80",
|
||||||
|
"basedir": "/usr/local/Cellar/mysql/5.7.20/",
|
||||||
|
"big_tables": "OFF",
|
||||||
|
"bind_address": "127.0.0.1",
|
||||||
|
"binlog_cache_size": "32768",
|
||||||
|
"binlog_checksum": "CRC32",
|
||||||
|
"binlog_direct_non_transactional_updates": "OFF",
|
||||||
|
"binlog_error_action": "ABORT_SERVER",
|
||||||
|
"binlog_format": "ROW",
|
||||||
|
"binlog_group_commit_sync_delay": "0",
|
||||||
|
"binlog_group_commit_sync_no_delay_count": "0",
|
||||||
|
"binlog_gtid_simple_recovery": "ON",
|
||||||
|
"binlog_max_flush_queue_time": "0",
|
||||||
|
"binlog_order_commits": "ON",
|
||||||
|
"binlog_row_image": "FULL",
|
||||||
|
"binlog_rows_query_log_events": "OFF",
|
||||||
|
"binlog_stmt_cache_size": "32768",
|
||||||
|
"block_encryption_mode": "aes-128-ecb",
|
||||||
|
"bulk_insert_buffer_size": "8388608",
|
||||||
|
"character_set_client": "utf8",
|
||||||
|
"character_set_connection": "utf8",
|
||||||
|
"character_set_database": "utf8",
|
||||||
|
"character_set_filesystem": "binary",
|
||||||
|
"character_set_results": "",
|
||||||
|
"character_set_server": "utf8",
|
||||||
|
"character_set_system": "utf8",
|
||||||
|
"character_sets_dir": "/usr/local/Cellar/mysql/5.7.20/share/mysql/charsets/",
|
||||||
|
"check_proxy_users": "OFF",
|
||||||
|
"collation_connection": "utf8_general_ci",
|
||||||
|
"collation_database": "utf8_general_ci",
|
||||||
|
"collation_server": "utf8_general_ci",
|
||||||
|
"completion_type": "NO_CHAIN",
|
||||||
|
"concurrent_insert": "AUTO",
|
||||||
|
"connect_timeout": "10",
|
||||||
|
"core_file": "OFF",
|
||||||
|
"datadir": "/usr/local/var/mysql/",
|
||||||
|
"date_format": "%Y-%m-%d",
|
||||||
|
"datetime_format": "%Y-%m-%d %H:%i:%s",
|
||||||
|
"default_authentication_plugin": "mysql_native_password",
|
||||||
|
"default_password_lifetime": "0",
|
||||||
|
"default_storage_engine": "InnoDB",
|
||||||
|
"default_tmp_storage_engine": "InnoDB",
|
||||||
|
"default_week_format": "0",
|
||||||
|
"delay_key_write": "ON",
|
||||||
|
"delayed_insert_limit": "100",
|
||||||
|
"delayed_insert_timeout": "300",
|
||||||
|
"delayed_queue_size": "1000",
|
||||||
|
"disabled_storage_engines": "",
|
||||||
|
"disconnect_on_expired_password": "ON",
|
||||||
|
"div_precision_increment": "4",
|
||||||
|
"end_markers_in_json": "OFF",
|
||||||
|
"enforce_gtid_consistency": "OFF",
|
||||||
|
"eq_range_index_dive_limit": "200",
|
||||||
|
"error_count": "0",
|
||||||
|
"event_scheduler": "OFF",
|
||||||
|
"expire_logs_days": "0",
|
||||||
|
"explicit_defaults_for_timestamp": "OFF",
|
||||||
|
"external_user": "",
|
||||||
|
"flush": "OFF",
|
||||||
|
"flush_time": "0",
|
||||||
|
"foreign_key_checks": "ON",
|
||||||
|
"ft_boolean_syntax": "+ -><()~*:\"\"&|",
|
||||||
|
"ft_max_word_len": "84",
|
||||||
|
"ft_min_word_len": "4",
|
||||||
|
"ft_query_expansion_limit": "20",
|
||||||
|
"ft_stopword_file": "(built-in)",
|
||||||
|
"general_log": "OFF",
|
||||||
|
"general_log_file": "/usr/local/var/mysql/JiangShuli.log",
|
||||||
|
"group_concat_max_len": "1024",
|
||||||
|
"gtid_executed_compression_period": "1000",
|
||||||
|
"gtid_mode": "OFF",
|
||||||
|
"gtid_next": "AUTOMATIC",
|
||||||
|
"gtid_owned": "",
|
||||||
|
"gtid_purged": "",
|
||||||
|
"have_compress": "YES",
|
||||||
|
"have_crypt": "YES",
|
||||||
|
"have_dynamic_loading": "YES",
|
||||||
|
"have_geometry": "YES",
|
||||||
|
"have_openssl": "YES",
|
||||||
|
"have_profiling": "YES",
|
||||||
|
"have_query_cache": "YES",
|
||||||
|
"have_rtree_keys": "YES",
|
||||||
|
"have_ssl": "YES",
|
||||||
|
"have_statement_timeout": "YES",
|
||||||
|
"have_symlink": "YES",
|
||||||
|
"host_cache_size": "279",
|
||||||
|
"hostname": "JiangShuli.local",
|
||||||
|
"identity": "0",
|
||||||
|
"ignore_builtin_innodb": "OFF",
|
||||||
|
"ignore_db_dirs": "",
|
||||||
|
"init_connect": "",
|
||||||
|
"init_file": "",
|
||||||
|
"init_slave": "",
|
||||||
|
"innodb_adaptive_flushing": "ON",
|
||||||
|
"innodb_adaptive_flushing_lwm": "10",
|
||||||
|
"innodb_adaptive_hash_index": "ON",
|
||||||
|
"innodb_adaptive_hash_index_parts": "8",
|
||||||
|
"innodb_adaptive_max_sleep_delay": "150000",
|
||||||
|
"innodb_api_bk_commit_interval": "5",
|
||||||
|
"innodb_api_disable_rowlock": "OFF",
|
||||||
|
"innodb_api_enable_binlog": "OFF",
|
||||||
|
"innodb_api_enable_mdl": "OFF",
|
||||||
|
"innodb_api_trx_level": "0",
|
||||||
|
"innodb_autoextend_increment": "64",
|
||||||
|
"innodb_autoinc_lock_mode": "1",
|
||||||
|
"innodb_buffer_pool_chunk_size": "134217728",
|
||||||
|
"innodb_buffer_pool_dump_at_shutdown": "ON",
|
||||||
|
"innodb_buffer_pool_dump_now": "OFF",
|
||||||
|
"innodb_buffer_pool_dump_pct": "25",
|
||||||
|
"innodb_buffer_pool_filename": "ib_buffer_pool",
|
||||||
|
"innodb_buffer_pool_instances": "1",
|
||||||
|
"innodb_buffer_pool_load_abort": "OFF",
|
||||||
|
"innodb_buffer_pool_load_at_startup": "ON",
|
||||||
|
"innodb_buffer_pool_load_now": "OFF",
|
||||||
|
"innodb_buffer_pool_size": "134217728",
|
||||||
|
"innodb_change_buffer_max_size": "25",
|
||||||
|
"innodb_change_buffering": "all",
|
||||||
|
"innodb_checksum_algorithm": "crc32",
|
||||||
|
"innodb_checksums": "ON",
|
||||||
|
"innodb_cmp_per_index_enabled": "OFF",
|
||||||
|
"innodb_commit_concurrency": "0",
|
||||||
|
"innodb_compression_failure_threshold_pct": "5",
|
||||||
|
"innodb_compression_level": "6",
|
||||||
|
"innodb_compression_pad_pct_max": "50",
|
||||||
|
"innodb_concurrency_tickets": "5000",
|
||||||
|
"innodb_data_file_path": "ibdata1:12M:autoextend",
|
||||||
|
"innodb_data_home_dir": "",
|
||||||
|
"innodb_deadlock_detect": "ON",
|
||||||
|
"innodb_default_row_format": "dynamic",
|
||||||
|
"innodb_disable_sort_file_cache": "OFF",
|
||||||
|
"innodb_doublewrite": "ON",
|
||||||
|
"innodb_fast_shutdown": "1",
|
||||||
|
"innodb_file_format": "Barracuda",
|
||||||
|
"innodb_file_format_check": "ON",
|
||||||
|
"innodb_file_format_max": "Barracuda",
|
||||||
|
"innodb_file_per_table": "ON",
|
||||||
|
"innodb_fill_factor": "100",
|
||||||
|
"innodb_flush_log_at_timeout": "1",
|
||||||
|
"innodb_flush_log_at_trx_commit": "1",
|
||||||
|
"innodb_flush_method": "",
|
||||||
|
"innodb_flush_neighbors": "1",
|
||||||
|
"innodb_flush_sync": "ON",
|
||||||
|
"innodb_flushing_avg_loops": "30",
|
||||||
|
"innodb_force_load_corrupted": "OFF",
|
||||||
|
"innodb_force_recovery": "0",
|
||||||
|
"innodb_ft_aux_table": "",
|
||||||
|
"innodb_ft_cache_size": "8000000",
|
||||||
|
"innodb_ft_enable_diag_print": "OFF",
|
||||||
|
"innodb_ft_enable_stopword": "ON",
|
||||||
|
"innodb_ft_max_token_size": "84",
|
||||||
|
"innodb_ft_min_token_size": "3",
|
||||||
|
"innodb_ft_num_word_optimize": "2000",
|
||||||
|
"innodb_ft_result_cache_limit": "2000000000",
|
||||||
|
"innodb_ft_server_stopword_table": "",
|
||||||
|
"innodb_ft_sort_pll_degree": "2",
|
||||||
|
"innodb_ft_total_cache_size": "640000000",
|
||||||
|
"innodb_ft_user_stopword_table": "",
|
||||||
|
"innodb_io_capacity": "200",
|
||||||
|
"innodb_io_capacity_max": "2000",
|
||||||
|
"innodb_large_prefix": "ON",
|
||||||
|
"innodb_lock_wait_timeout": "50",
|
||||||
|
"innodb_locks_unsafe_for_binlog": "OFF",
|
||||||
|
"innodb_log_buffer_size": "16777216",
|
||||||
|
"innodb_log_checksums": "ON",
|
||||||
|
"innodb_log_compressed_pages": "ON",
|
||||||
|
"innodb_log_file_size": "50331648",
|
||||||
|
"innodb_log_files_in_group": "2",
|
||||||
|
"innodb_log_group_home_dir": "./",
|
||||||
|
"innodb_log_write_ahead_size": "8192",
|
||||||
|
"innodb_lru_scan_depth": "1024",
|
||||||
|
"innodb_max_dirty_pages_pct": "75.000000",
|
||||||
|
"innodb_max_dirty_pages_pct_lwm": "0.000000",
|
||||||
|
"innodb_max_purge_lag": "0",
|
||||||
|
"innodb_max_purge_lag_delay": "0",
|
||||||
|
"innodb_max_undo_log_size": "1073741824",
|
||||||
|
"innodb_monitor_disable": "",
|
||||||
|
"innodb_monitor_enable": "",
|
||||||
|
"innodb_monitor_reset": "",
|
||||||
|
"innodb_monitor_reset_all": "",
|
||||||
|
"innodb_old_blocks_pct": "37",
|
||||||
|
"innodb_old_blocks_time": "1000",
|
||||||
|
"innodb_online_alter_log_max_size": "134217728",
|
||||||
|
"innodb_open_files": "2000",
|
||||||
|
"innodb_optimize_fulltext_only": "OFF",
|
||||||
|
"innodb_page_cleaners": "1",
|
||||||
|
"innodb_page_size": "16384",
|
||||||
|
"innodb_print_all_deadlocks": "OFF",
|
||||||
|
"innodb_purge_batch_size": "300",
|
||||||
|
"innodb_purge_rseg_truncate_frequency": "128",
|
||||||
|
"innodb_purge_threads": "4",
|
||||||
|
"innodb_random_read_ahead": "OFF",
|
||||||
|
"innodb_read_ahead_threshold": "56",
|
||||||
|
"innodb_read_io_threads": "4",
|
||||||
|
"innodb_read_only": "OFF",
|
||||||
|
"innodb_replication_delay": "0",
|
||||||
|
"innodb_rollback_on_timeout": "OFF",
|
||||||
|
"innodb_rollback_segments": "128",
|
||||||
|
"innodb_sort_buffer_size": "1048576",
|
||||||
|
"innodb_spin_wait_delay": "6",
|
||||||
|
"innodb_stats_auto_recalc": "ON",
|
||||||
|
"innodb_stats_include_delete_marked": "OFF",
|
||||||
|
"innodb_stats_method": "nulls_equal",
|
||||||
|
"innodb_stats_on_metadata": "OFF",
|
||||||
|
"innodb_stats_persistent": "ON",
|
||||||
|
"innodb_stats_persistent_sample_pages": "20",
|
||||||
|
"innodb_stats_sample_pages": "8",
|
||||||
|
"innodb_stats_transient_sample_pages": "8",
|
||||||
|
"innodb_status_output": "OFF",
|
||||||
|
"innodb_status_output_locks": "OFF",
|
||||||
|
"innodb_strict_mode": "ON",
|
||||||
|
"innodb_support_xa": "ON",
|
||||||
|
"innodb_sync_array_size": "1",
|
||||||
|
"innodb_sync_spin_loops": "30",
|
||||||
|
"innodb_table_locks": "ON",
|
||||||
|
"innodb_temp_data_file_path": "ibtmp1:12M:autoextend",
|
||||||
|
"innodb_thread_concurrency": "0",
|
||||||
|
"innodb_thread_sleep_delay": "10000",
|
||||||
|
"innodb_tmpdir": "",
|
||||||
|
"innodb_undo_directory": "./",
|
||||||
|
"innodb_undo_log_truncate": "OFF",
|
||||||
|
"innodb_undo_logs": "128",
|
||||||
|
"innodb_undo_tablespaces": "0",
|
||||||
|
"innodb_use_native_aio": "OFF",
|
||||||
|
"innodb_version": "5.7.20",
|
||||||
|
"innodb_write_io_threads": "4",
|
||||||
|
"insert_id": "0",
|
||||||
|
"interactive_timeout": "28800",
|
||||||
|
"internal_tmp_disk_storage_engine": "InnoDB",
|
||||||
|
"join_buffer_size": "262144",
|
||||||
|
"keep_files_on_create": "OFF",
|
||||||
|
"key_buffer_size": "8388608",
|
||||||
|
"key_cache_age_threshold": "300",
|
||||||
|
"key_cache_block_size": "1024",
|
||||||
|
"key_cache_division_limit": "100",
|
||||||
|
"large_files_support": "ON",
|
||||||
|
"large_page_size": "0",
|
||||||
|
"large_pages": "OFF",
|
||||||
|
"last_insert_id": "0",
|
||||||
|
"lc_messages": "en_US",
|
||||||
|
"lc_messages_dir": "/usr/local/Cellar/mysql/5.7.20/share/mysql/",
|
||||||
|
"lc_time_names": "en_US",
|
||||||
|
"license": "GPL",
|
||||||
|
"local_infile": "ON",
|
||||||
|
"lock_wait_timeout": "31536000",
|
||||||
|
"locked_in_memory": "OFF",
|
||||||
|
"log_bin": "OFF",
|
||||||
|
"log_bin_basename": "",
|
||||||
|
"log_bin_index": "",
|
||||||
|
"log_bin_trust_function_creators": "OFF",
|
||||||
|
"log_bin_use_v1_row_events": "OFF",
|
||||||
|
"log_builtin_as_identified_by_password": "OFF",
|
||||||
|
"log_error": "./JiangShuli.local.err",
|
||||||
|
"log_error_verbosity": "3",
|
||||||
|
"log_output": "FILE",
|
||||||
|
"log_queries_not_using_indexes": "OFF",
|
||||||
|
"log_slave_updates": "OFF",
|
||||||
|
"log_slow_admin_statements": "OFF",
|
||||||
|
"log_slow_slave_statements": "OFF",
|
||||||
|
"log_statements_unsafe_for_binlog": "ON",
|
||||||
|
"log_syslog": "OFF",
|
||||||
|
"log_syslog_facility": "daemon",
|
||||||
|
"log_syslog_include_pid": "ON",
|
||||||
|
"log_syslog_tag": "",
|
||||||
|
"log_throttle_queries_not_using_indexes": "0",
|
||||||
|
"log_timestamps": "UTC",
|
||||||
|
"log_warnings": "2",
|
||||||
|
"long_query_time": "10.000000",
|
||||||
|
"low_priority_updates": "OFF",
|
||||||
|
"lower_case_file_system": "ON",
|
||||||
|
"lower_case_table_names": "2",
|
||||||
|
"master_info_repository": "FILE",
|
||||||
|
"master_verify_checksum": "OFF",
|
||||||
|
"max_allowed_packet": "4194304",
|
||||||
|
"max_binlog_cache_size": "18446744073709547520",
|
||||||
|
"max_binlog_size": "1073741824",
|
||||||
|
"max_binlog_stmt_cache_size": "18446744073709547520",
|
||||||
|
"max_connect_errors": "100",
|
||||||
|
"max_connections": "151",
|
||||||
|
"max_delayed_threads": "20",
|
||||||
|
"max_digest_length": "1024",
|
||||||
|
"max_error_count": "64",
|
||||||
|
"max_execution_time": "0",
|
||||||
|
"max_heap_table_size": "16777216",
|
||||||
|
"max_insert_delayed_threads": "20",
|
||||||
|
"max_join_size": "18446744073709551615",
|
||||||
|
"max_length_for_sort_data": "1024",
|
||||||
|
"max_points_in_geometry": "65536",
|
||||||
|
"max_prepared_stmt_count": "16382",
|
||||||
|
"max_relay_log_size": "0",
|
||||||
|
"max_seeks_for_key": "18446744073709551615",
|
||||||
|
"max_sort_length": "1024",
|
||||||
|
"max_sp_recursion_depth": "0",
|
||||||
|
"max_tmp_tables": "32",
|
||||||
|
"max_user_connections": "0",
|
||||||
|
"max_write_lock_count": "18446744073709551615",
|
||||||
|
"metadata_locks_cache_size": "1024",
|
||||||
|
"metadata_locks_hash_instances": "8",
|
||||||
|
"min_examined_row_limit": "0",
|
||||||
|
"multi_range_count": "256",
|
||||||
|
"myisam_data_pointer_size": "6",
|
||||||
|
"myisam_max_sort_file_size": "9223372036853727232",
|
||||||
|
"myisam_mmap_size": "18446744073709551615",
|
||||||
|
"myisam_recover_options": "OFF",
|
||||||
|
"myisam_repair_threads": "1",
|
||||||
|
"myisam_sort_buffer_size": "8388608",
|
||||||
|
"myisam_stats_method": "nulls_unequal",
|
||||||
|
"myisam_use_mmap": "OFF",
|
||||||
|
"mysql_native_password_proxy_users": "OFF",
|
||||||
|
"net_buffer_length": "16384",
|
||||||
|
"net_read_timeout": "30",
|
||||||
|
"net_retry_count": "10",
|
||||||
|
"net_write_timeout": "60",
|
||||||
|
"new": "OFF",
|
||||||
|
"ngram_token_size": "2",
|
||||||
|
"offline_mode": "OFF",
|
||||||
|
"old": "OFF",
|
||||||
|
"old_alter_table": "OFF",
|
||||||
|
"old_passwords": "0",
|
||||||
|
"open_files_limit": "5000",
|
||||||
|
"optimizer_prune_level": "1",
|
||||||
|
"optimizer_search_depth": "62",
|
||||||
|
"optimizer_switch": "index_merge=on,index_merge_union=on,index_merge_sort_union=on,index_merge_intersection=on,engine_condition_pushdown=on,index_condition_pushdown=on,mrr=on,mrr_cost_based=on,block_nested_loop=on,batched_key_access=off,materialization=on,semijoin=on,loosescan=on,firstmatch=on,duplicateweedout=on,subquery_materialization_cost_based=on,use_index_extensions=on,condition_fanout_filter=on,derived_merge=on",
|
||||||
|
"optimizer_trace": "enabled=off,one_line=off",
|
||||||
|
"optimizer_trace_features": "greedy_search=on,range_optimizer=on,dynamic_range=on,repeated_subselect=on",
|
||||||
|
"optimizer_trace_limit": "1",
|
||||||
|
"optimizer_trace_max_mem_size": "16384",
|
||||||
|
"optimizer_trace_offset": "-1",
|
||||||
|
"parser_max_mem_size": "18446744073709551615",
|
||||||
|
"performance_schema": "ON",
|
||||||
|
"performance_schema_accounts_size": "-1",
|
||||||
|
"performance_schema_digests_size": "10000",
|
||||||
|
"performance_schema_events_stages_history_long_size": "10000",
|
||||||
|
"performance_schema_events_stages_history_size": "10",
|
||||||
|
"performance_schema_events_statements_history_long_size": "10000",
|
||||||
|
"performance_schema_events_statements_history_size": "10",
|
||||||
|
"performance_schema_events_transactions_history_long_size": "10000",
|
||||||
|
"performance_schema_events_transactions_history_size": "10",
|
||||||
|
"performance_schema_events_waits_history_long_size": "10000",
|
||||||
|
"performance_schema_events_waits_history_size": "10",
|
||||||
|
"performance_schema_hosts_size": "-1",
|
||||||
|
"performance_schema_max_cond_classes": "80",
|
||||||
|
"performance_schema_max_cond_instances": "-1",
|
||||||
|
"performance_schema_max_digest_length": "1024",
|
||||||
|
"performance_schema_max_file_classes": "80",
|
||||||
|
"performance_schema_max_file_handles": "32768",
|
||||||
|
"performance_schema_max_file_instances": "-1",
|
||||||
|
"performance_schema_max_index_stat": "-1",
|
||||||
|
"performance_schema_max_memory_classes": "320",
|
||||||
|
"performance_schema_max_metadata_locks": "-1",
|
||||||
|
"performance_schema_max_mutex_classes": "210",
|
||||||
|
"performance_schema_max_mutex_instances": "-1",
|
||||||
|
"performance_schema_max_prepared_statements_instances": "-1",
|
||||||
|
"performance_schema_max_program_instances": "-1",
|
||||||
|
"performance_schema_max_rwlock_classes": "40",
|
||||||
|
"performance_schema_max_rwlock_instances": "-1",
|
||||||
|
"performance_schema_max_socket_classes": "10",
|
||||||
|
"performance_schema_max_socket_instances": "-1",
|
||||||
|
"performance_schema_max_sql_text_length": "1024",
|
||||||
|
"performance_schema_max_stage_classes": "150",
|
||||||
|
"performance_schema_max_statement_classes": "193",
|
||||||
|
"performance_schema_max_statement_stack": "10",
|
||||||
|
"performance_schema_max_table_handles": "-1",
|
||||||
|
"performance_schema_max_table_instances": "-1",
|
||||||
|
"performance_schema_max_table_lock_stat": "-1",
|
||||||
|
"performance_schema_max_thread_classes": "50",
|
||||||
|
"performance_schema_max_thread_instances": "-1",
|
||||||
|
"performance_schema_session_connect_attrs_size": "512",
|
||||||
|
"performance_schema_setup_actors_size": "-1",
|
||||||
|
"performance_schema_setup_objects_size": "-1",
|
||||||
|
"performance_schema_users_size": "-1",
|
||||||
|
"pid_file": "/usr/local/var/mysql/JiangShuli.local.pid",
|
||||||
|
"plugin_dir": "/usr/local/Cellar/mysql/5.7.20/lib/plugin/",
|
||||||
|
"port": "3306",
|
||||||
|
"preload_buffer_size": "32768",
|
||||||
|
"profiling": "OFF",
|
||||||
|
"profiling_history_size": "15",
|
||||||
|
"protocol_version": "10",
|
||||||
|
"proxy_user": "",
|
||||||
|
"pseudo_slave_mode": "OFF",
|
||||||
|
"pseudo_thread_id": "3",
|
||||||
|
"query_alloc_block_size": "8192",
|
||||||
|
"query_cache_limit": "1048576",
|
||||||
|
"query_cache_min_res_unit": "4096",
|
||||||
|
"query_cache_size": "1048576",
|
||||||
|
"query_cache_type": "OFF",
|
||||||
|
"query_cache_wlock_invalidate": "OFF",
|
||||||
|
"query_prealloc_size": "8192",
|
||||||
|
"rand_seed1": "0",
|
||||||
|
"rand_seed2": "0",
|
||||||
|
"range_alloc_block_size": "4096",
|
||||||
|
"range_optimizer_max_mem_size": "8388608",
|
||||||
|
"rbr_exec_mode": "STRICT",
|
||||||
|
"read_buffer_size": "131072",
|
||||||
|
"read_only": "OFF",
|
||||||
|
"read_rnd_buffer_size": "262144",
|
||||||
|
"relay_log": "",
|
||||||
|
"relay_log_basename": "/usr/local/var/mysql/JiangShuli-relay-bin",
|
||||||
|
"relay_log_index": "/usr/local/var/mysql/JiangShuli-relay-bin.index",
|
||||||
|
"relay_log_info_file": "relay-log.info",
|
||||||
|
"relay_log_info_repository": "FILE",
|
||||||
|
"relay_log_purge": "ON",
|
||||||
|
"relay_log_recovery": "OFF",
|
||||||
|
"relay_log_space_limit": "0",
|
||||||
|
"report_host": "",
|
||||||
|
"report_password": "",
|
||||||
|
"report_port": "3306",
|
||||||
|
"report_user": "",
|
||||||
|
"require_secure_transport": "OFF",
|
||||||
|
"rpl_stop_slave_timeout": "31536000",
|
||||||
|
"secure_auth": "ON",
|
||||||
|
"secure_file_priv": "NULL",
|
||||||
|
"server_id": "0",
|
||||||
|
"server_id_bits": "32",
|
||||||
|
"server_uuid": "89286c98-d4ae-11e7-bb02-defe5cd105a2",
|
||||||
|
"session_track_gtids": "OFF",
|
||||||
|
"session_track_schema": "ON",
|
||||||
|
"session_track_state_change": "OFF",
|
||||||
|
"session_track_system_variables": "time_zone,autocommit,character_set_client,character_set_results,character_set_connection",
|
||||||
|
"session_track_transaction_info": "OFF",
|
||||||
|
"sha256_password_auto_generate_rsa_keys": "ON",
|
||||||
|
"sha256_password_private_key_path": "private_key.pem",
|
||||||
|
"sha256_password_proxy_users": "OFF",
|
||||||
|
"sha256_password_public_key_path": "public_key.pem",
|
||||||
|
"show_compatibility_56": "OFF",
|
||||||
|
"show_old_temporals": "OFF",
|
||||||
|
"skip_external_locking": "ON",
|
||||||
|
"skip_name_resolve": "OFF",
|
||||||
|
"skip_networking": "OFF",
|
||||||
|
"skip_show_database": "OFF",
|
||||||
|
"slave_allow_batching": "OFF",
|
||||||
|
"slave_checkpoint_group": "512",
|
||||||
|
"slave_checkpoint_period": "300",
|
||||||
|
"slave_compressed_protocol": "OFF",
|
||||||
|
"slave_exec_mode": "STRICT",
|
||||||
|
"slave_load_tmpdir": "/var/folders/hd/lw_b86v52pdfg323g_jn76k80000gn/T/",
|
||||||
|
"slave_max_allowed_packet": "1073741824",
|
||||||
|
"slave_net_timeout": "60",
|
||||||
|
"slave_parallel_type": "DATABASE",
|
||||||
|
"slave_parallel_workers": "0",
|
||||||
|
"slave_pending_jobs_size_max": "16777216",
|
||||||
|
"slave_preserve_commit_order": "OFF",
|
||||||
|
"slave_rows_search_algorithms": "TABLE_SCAN,INDEX_SCAN",
|
||||||
|
"slave_skip_errors": "OFF",
|
||||||
|
"slave_sql_verify_checksum": "ON",
|
||||||
|
"slave_transaction_retries": "10",
|
||||||
|
"slave_type_conversions": "",
|
||||||
|
"slow_launch_time": "2",
|
||||||
|
"slow_query_log": "OFF",
|
||||||
|
"slow_query_log_file": "/usr/local/var/mysql/JiangShuli-slow.log",
|
||||||
|
"socket": "/tmp/mysql.sock",
|
||||||
|
"sort_buffer_size": "262144",
|
||||||
|
"sql_auto_is_null": "OFF",
|
||||||
|
"sql_big_selects": "ON",
|
||||||
|
"sql_buffer_result": "OFF",
|
||||||
|
"sql_log_bin": "ON",
|
||||||
|
"sql_log_off": "OFF",
|
||||||
|
"sql_mode": "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION",
|
||||||
|
"sql_notes": "ON",
|
||||||
|
"sql_quote_show_create": "ON",
|
||||||
|
"sql_safe_updates": "OFF",
|
||||||
|
"sql_select_limit": "18446744073709551615",
|
||||||
|
"sql_slave_skip_counter": "0",
|
||||||
|
"sql_warnings": "OFF",
|
||||||
|
"ssl_ca": "ca.pem",
|
||||||
|
"ssl_capath": "",
|
||||||
|
"ssl_cert": "server-cert.pem",
|
||||||
|
"ssl_cipher": "",
|
||||||
|
"ssl_crl": "",
|
||||||
|
"ssl_crlpath": "",
|
||||||
|
"ssl_key": "server-key.pem",
|
||||||
|
"stored_program_cache": "256",
|
||||||
|
"super_read_only": "OFF",
|
||||||
|
"sync_binlog": "1",
|
||||||
|
"sync_frm": "ON",
|
||||||
|
"sync_master_info": "10000",
|
||||||
|
"sync_relay_log": "10000",
|
||||||
|
"sync_relay_log_info": "10000",
|
||||||
|
"system_time_zone": "EST",
|
||||||
|
"table_definition_cache": "1400",
|
||||||
|
"table_open_cache": "2000",
|
||||||
|
"table_open_cache_instances": "16",
|
||||||
|
"thread_cache_size": "9",
|
||||||
|
"thread_handling": "one-thread-per-connection",
|
||||||
|
"thread_stack": "262144",
|
||||||
|
"time_format": "%H:%i:%s",
|
||||||
|
"time_zone": "SYSTEM",
|
||||||
|
"timestamp": "1512532880.980241",
|
||||||
|
"tls_version": "TLSv1,TLSv1.1,TLSv1.2",
|
||||||
|
"tmp_table_size": "16777216",
|
||||||
|
"tmpdir": "/var/folders/hd/lw_b86v52pdfg323g_jn76k80000gn/T/",
|
||||||
|
"transaction_alloc_block_size": "8192",
|
||||||
|
"transaction_allow_batching": "OFF",
|
||||||
|
"transaction_isolation": "REPEATABLE-READ",
|
||||||
|
"transaction_prealloc_size": "4096",
|
||||||
|
"transaction_read_only": "OFF",
|
||||||
|
"transaction_write_set_extraction": "OFF",
|
||||||
|
"tx_isolation": "REPEATABLE-READ",
|
||||||
|
"tx_read_only": "OFF",
|
||||||
|
"unique_checks": "ON",
|
||||||
|
"updatable_views_with_limit": "YES",
|
||||||
|
"version": "5.7.20",
|
||||||
|
"version_comment": "Homebrew",
|
||||||
|
"version_compile_machine": "x86_64",
|
||||||
|
"version_compile_os": "osx10.12",
|
||||||
|
"wait_timeout": "28800",
|
||||||
|
"warning_count": "0"
|
||||||
|
}},
|
||||||
|
"local": null
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,362 @@
|
||||||
|
{
|
||||||
|
"global": {"global": {
|
||||||
|
"aborted_clients": "0",
|
||||||
|
"aborted_connects": "0",
|
||||||
|
"binlog_cache_disk_use": "0",
|
||||||
|
"binlog_cache_use": "0",
|
||||||
|
"binlog_stmt_cache_disk_use": "0",
|
||||||
|
"binlog_stmt_cache_use": "0",
|
||||||
|
"bytes_received": "1227",
|
||||||
|
"bytes_sent": "29713",
|
||||||
|
"com_admin_commands": "0",
|
||||||
|
"com_alter_db": "0",
|
||||||
|
"com_alter_db_upgrade": "0",
|
||||||
|
"com_alter_event": "0",
|
||||||
|
"com_alter_function": "0",
|
||||||
|
"com_alter_instance": "0",
|
||||||
|
"com_alter_procedure": "0",
|
||||||
|
"com_alter_server": "0",
|
||||||
|
"com_alter_table": "0",
|
||||||
|
"com_alter_tablespace": "0",
|
||||||
|
"com_alter_user": "0",
|
||||||
|
"com_analyze": "0",
|
||||||
|
"com_assign_to_keycache": "0",
|
||||||
|
"com_begin": "0",
|
||||||
|
"com_binlog": "0",
|
||||||
|
"com_call_procedure": "0",
|
||||||
|
"com_change_db": "0",
|
||||||
|
"com_change_master": "0",
|
||||||
|
"com_change_repl_filter": "0",
|
||||||
|
"com_check": "0",
|
||||||
|
"com_checksum": "0",
|
||||||
|
"com_commit": "0",
|
||||||
|
"com_create_db": "0",
|
||||||
|
"com_create_event": "0",
|
||||||
|
"com_create_function": "0",
|
||||||
|
"com_create_index": "0",
|
||||||
|
"com_create_procedure": "0",
|
||||||
|
"com_create_server": "0",
|
||||||
|
"com_create_table": "0",
|
||||||
|
"com_create_trigger": "0",
|
||||||
|
"com_create_udf": "0",
|
||||||
|
"com_create_user": "0",
|
||||||
|
"com_create_view": "0",
|
||||||
|
"com_dealloc_sql": "0",
|
||||||
|
"com_delete": "0",
|
||||||
|
"com_delete_multi": "0",
|
||||||
|
"com_do": "0",
|
||||||
|
"com_drop_db": "0",
|
||||||
|
"com_drop_event": "0",
|
||||||
|
"com_drop_function": "0",
|
||||||
|
"com_drop_index": "0",
|
||||||
|
"com_drop_procedure": "0",
|
||||||
|
"com_drop_server": "0",
|
||||||
|
"com_drop_table": "0",
|
||||||
|
"com_drop_trigger": "0",
|
||||||
|
"com_drop_user": "0",
|
||||||
|
"com_drop_view": "0",
|
||||||
|
"com_empty_query": "0",
|
||||||
|
"com_execute_sql": "0",
|
||||||
|
"com_explain_other": "0",
|
||||||
|
"com_flush": "0",
|
||||||
|
"com_get_diagnostics": "0",
|
||||||
|
"com_grant": "0",
|
||||||
|
"com_group_replication_start": "0",
|
||||||
|
"com_group_replication_stop": "0",
|
||||||
|
"com_ha_close": "0",
|
||||||
|
"com_ha_open": "0",
|
||||||
|
"com_ha_read": "0",
|
||||||
|
"com_help": "0",
|
||||||
|
"com_insert": "0",
|
||||||
|
"com_insert_select": "0",
|
||||||
|
"com_install_plugin": "0",
|
||||||
|
"com_kill": "0",
|
||||||
|
"com_load": "0",
|
||||||
|
"com_lock_tables": "0",
|
||||||
|
"com_optimize": "0",
|
||||||
|
"com_preload_keys": "0",
|
||||||
|
"com_prepare_sql": "0",
|
||||||
|
"com_purge": "0",
|
||||||
|
"com_purge_before_date": "0",
|
||||||
|
"com_release_savepoint": "0",
|
||||||
|
"com_rename_table": "0",
|
||||||
|
"com_rename_user": "0",
|
||||||
|
"com_repair": "0",
|
||||||
|
"com_replace": "0",
|
||||||
|
"com_replace_select": "0",
|
||||||
|
"com_reset": "0",
|
||||||
|
"com_resignal": "0",
|
||||||
|
"com_revoke": "0",
|
||||||
|
"com_revoke_all": "0",
|
||||||
|
"com_rollback": "0",
|
||||||
|
"com_rollback_to_savepoint": "0",
|
||||||
|
"com_savepoint": "0",
|
||||||
|
"com_select": "2",
|
||||||
|
"com_set_option": "2",
|
||||||
|
"com_show_binlog_events": "0",
|
||||||
|
"com_show_binlogs": "0",
|
||||||
|
"com_show_charsets": "1",
|
||||||
|
"com_show_collations": "1",
|
||||||
|
"com_show_create_db": "0",
|
||||||
|
"com_show_create_event": "0",
|
||||||
|
"com_show_create_func": "0",
|
||||||
|
"com_show_create_proc": "0",
|
||||||
|
"com_show_create_table": "0",
|
||||||
|
"com_show_create_trigger": "0",
|
||||||
|
"com_show_create_user": "0",
|
||||||
|
"com_show_databases": "0",
|
||||||
|
"com_show_engine_logs": "0",
|
||||||
|
"com_show_engine_mutex": "0",
|
||||||
|
"com_show_engine_status": "0",
|
||||||
|
"com_show_errors": "0",
|
||||||
|
"com_show_events": "0",
|
||||||
|
"com_show_fields": "0",
|
||||||
|
"com_show_function_code": "0",
|
||||||
|
"com_show_function_status": "0",
|
||||||
|
"com_show_grants": "0",
|
||||||
|
"com_show_keys": "0",
|
||||||
|
"com_show_master_status": "0",
|
||||||
|
"com_show_open_tables": "0",
|
||||||
|
"com_show_plugins": "0",
|
||||||
|
"com_show_privileges": "0",
|
||||||
|
"com_show_procedure_code": "0",
|
||||||
|
"com_show_procedure_status": "0",
|
||||||
|
"com_show_processlist": "0",
|
||||||
|
"com_show_profile": "0",
|
||||||
|
"com_show_profiles": "0",
|
||||||
|
"com_show_relaylog_events": "0",
|
||||||
|
"com_show_slave_hosts": "0",
|
||||||
|
"com_show_slave_status": "0",
|
||||||
|
"com_show_status": "1",
|
||||||
|
"com_show_storage_engines": "0",
|
||||||
|
"com_show_table_status": "0",
|
||||||
|
"com_show_tables": "0",
|
||||||
|
"com_show_triggers": "0",
|
||||||
|
"com_show_variables": "2",
|
||||||
|
"com_show_warnings": "0",
|
||||||
|
"com_shutdown": "0",
|
||||||
|
"com_signal": "0",
|
||||||
|
"com_slave_start": "0",
|
||||||
|
"com_slave_stop": "0",
|
||||||
|
"com_stmt_close": "0",
|
||||||
|
"com_stmt_execute": "0",
|
||||||
|
"com_stmt_fetch": "0",
|
||||||
|
"com_stmt_prepare": "0",
|
||||||
|
"com_stmt_reprepare": "0",
|
||||||
|
"com_stmt_reset": "0",
|
||||||
|
"com_stmt_send_long_data": "0",
|
||||||
|
"com_truncate": "0",
|
||||||
|
"com_uninstall_plugin": "0",
|
||||||
|
"com_unlock_tables": "0",
|
||||||
|
"com_update": "0",
|
||||||
|
"com_update_multi": "0",
|
||||||
|
"com_xa_commit": "0",
|
||||||
|
"com_xa_end": "0",
|
||||||
|
"com_xa_prepare": "0",
|
||||||
|
"com_xa_recover": "0",
|
||||||
|
"com_xa_rollback": "0",
|
||||||
|
"com_xa_start": "0",
|
||||||
|
"compression": "OFF",
|
||||||
|
"connection_errors_accept": "0",
|
||||||
|
"connection_errors_internal": "0",
|
||||||
|
"connection_errors_max_connections": "0",
|
||||||
|
"connection_errors_peer_address": "0",
|
||||||
|
"connection_errors_select": "0",
|
||||||
|
"connection_errors_tcpwrap": "0",
|
||||||
|
"connections": "4",
|
||||||
|
"created_tmp_disk_tables": "0",
|
||||||
|
"created_tmp_files": "5",
|
||||||
|
"created_tmp_tables": "4",
|
||||||
|
"delayed_errors": "0",
|
||||||
|
"delayed_insert_threads": "0",
|
||||||
|
"delayed_writes": "0",
|
||||||
|
"flush_commands": "1",
|
||||||
|
"handler_commit": "0",
|
||||||
|
"handler_delete": "0",
|
||||||
|
"handler_discover": "0",
|
||||||
|
"handler_external_lock": "4",
|
||||||
|
"handler_mrr_init": "0",
|
||||||
|
"handler_prepare": "0",
|
||||||
|
"handler_read_first": "0",
|
||||||
|
"handler_read_key": "0",
|
||||||
|
"handler_read_last": "0",
|
||||||
|
"handler_read_next": "0",
|
||||||
|
"handler_read_prev": "0",
|
||||||
|
"handler_read_rnd": "0",
|
||||||
|
"handler_read_rnd_next": "2317",
|
||||||
|
"handler_rollback": "0",
|
||||||
|
"handler_savepoint": "0",
|
||||||
|
"handler_savepoint_rollback": "0",
|
||||||
|
"handler_update": "0",
|
||||||
|
"handler_write": "1287",
|
||||||
|
"innodb_available_undo_logs": "128",
|
||||||
|
"innodb_buffer_pool_bytes_data": "11288576",
|
||||||
|
"innodb_buffer_pool_bytes_dirty": "0",
|
||||||
|
"innodb_buffer_pool_dump_status": "Dumping of buffer pool not started",
|
||||||
|
"innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 171205 23:01:14",
|
||||||
|
"innodb_buffer_pool_pages_data": "689",
|
||||||
|
"innodb_buffer_pool_pages_dirty": "0",
|
||||||
|
"innodb_buffer_pool_pages_flushed": "37",
|
||||||
|
"innodb_buffer_pool_pages_free": "7502",
|
||||||
|
"innodb_buffer_pool_pages_misc": "0",
|
||||||
|
"innodb_buffer_pool_pages_total": "8191",
|
||||||
|
"innodb_buffer_pool_read_ahead": "0",
|
||||||
|
"innodb_buffer_pool_read_ahead_evicted": "0",
|
||||||
|
"innodb_buffer_pool_read_ahead_rnd": "0",
|
||||||
|
"innodb_buffer_pool_read_requests": "1833",
|
||||||
|
"innodb_buffer_pool_reads": "655",
|
||||||
|
"innodb_buffer_pool_resize_status": "",
|
||||||
|
"innodb_buffer_pool_wait_free": "0",
|
||||||
|
"innodb_buffer_pool_write_requests": "515",
|
||||||
|
"innodb_data_fsyncs": "7",
|
||||||
|
"innodb_data_pending_fsyncs": "0",
|
||||||
|
"innodb_data_pending_reads": "0",
|
||||||
|
"innodb_data_pending_writes": "0",
|
||||||
|
"innodb_data_read": "10801664",
|
||||||
|
"innodb_data_reads": "683",
|
||||||
|
"innodb_data_writes": "54",
|
||||||
|
"innodb_data_written": "641024",
|
||||||
|
"innodb_dblwr_pages_written": "2",
|
||||||
|
"innodb_dblwr_writes": "1",
|
||||||
|
"innodb_log_waits": "0",
|
||||||
|
"innodb_log_write_requests": "0",
|
||||||
|
"innodb_log_writes": "2",
|
||||||
|
"innodb_num_open_files": "20",
|
||||||
|
"innodb_os_log_fsyncs": "4",
|
||||||
|
"innodb_os_log_pending_fsyncs": "0",
|
||||||
|
"innodb_os_log_pending_writes": "0",
|
||||||
|
"innodb_os_log_written": "1024",
|
||||||
|
"innodb_page_size": "16384",
|
||||||
|
"innodb_pages_created": "35",
|
||||||
|
"innodb_pages_read": "654",
|
||||||
|
"innodb_pages_written": "37",
|
||||||
|
"innodb_row_lock_current_waits": "0",
|
||||||
|
"innodb_row_lock_time": "0",
|
||||||
|
"innodb_row_lock_time_avg": "0",
|
||||||
|
"innodb_row_lock_time_max": "0",
|
||||||
|
"innodb_row_lock_waits": "0",
|
||||||
|
"innodb_rows_deleted": "0",
|
||||||
|
"innodb_rows_inserted": "0",
|
||||||
|
"innodb_rows_read": "8",
|
||||||
|
"innodb_rows_updated": "0",
|
||||||
|
"innodb_truncated_status_writes": "0",
|
||||||
|
"key_blocks_not_flushed": "0",
|
||||||
|
"key_blocks_unused": "6695",
|
||||||
|
"key_blocks_used": "3",
|
||||||
|
"key_read_requests": "6",
|
||||||
|
"key_reads": "3",
|
||||||
|
"key_write_requests": "0",
|
||||||
|
"key_writes": "0",
|
||||||
|
"last_query_cost": "0.000000",
|
||||||
|
"last_query_partial_plans": "2",
|
||||||
|
"locked_connects": "0",
|
||||||
|
"max_execution_time_exceeded": "0",
|
||||||
|
"max_execution_time_set": "0",
|
||||||
|
"max_execution_time_set_failed": "0",
|
||||||
|
"max_used_connections": "1",
|
||||||
|
"max_used_connections_time": "2017-12-05 23:01:20",
|
||||||
|
"not_flushed_delayed_rows": "0",
|
||||||
|
"ongoing_anonymous_transaction_count": "0",
|
||||||
|
"open_files": "14",
|
||||||
|
"open_streams": "0",
|
||||||
|
"open_table_definitions": "228",
|
||||||
|
"open_tables": "100",
|
||||||
|
"opened_files": "357",
|
||||||
|
"opened_table_definitions": "0",
|
||||||
|
"opened_tables": "1",
|
||||||
|
"performance_schema_accounts_lost": "0",
|
||||||
|
"performance_schema_cond_classes_lost": "0",
|
||||||
|
"performance_schema_cond_instances_lost": "0",
|
||||||
|
"performance_schema_digest_lost": "0",
|
||||||
|
"performance_schema_file_classes_lost": "0",
|
||||||
|
"performance_schema_file_handles_lost": "0",
|
||||||
|
"performance_schema_file_instances_lost": "0",
|
||||||
|
"performance_schema_hosts_lost": "0",
|
||||||
|
"performance_schema_index_stat_lost": "0",
|
||||||
|
"performance_schema_locker_lost": "0",
|
||||||
|
"performance_schema_memory_classes_lost": "0",
|
||||||
|
"performance_schema_metadata_lock_lost": "0",
|
||||||
|
"performance_schema_mutex_classes_lost": "0",
|
||||||
|
"performance_schema_mutex_instances_lost": "0",
|
||||||
|
"performance_schema_nested_statement_lost": "0",
|
||||||
|
"performance_schema_prepared_statements_lost": "0",
|
||||||
|
"performance_schema_program_lost": "0",
|
||||||
|
"performance_schema_rwlock_classes_lost": "0",
|
||||||
|
"performance_schema_rwlock_instances_lost": "0",
|
||||||
|
"performance_schema_session_connect_attrs_lost": "0",
|
||||||
|
"performance_schema_socket_classes_lost": "0",
|
||||||
|
"performance_schema_socket_instances_lost": "0",
|
||||||
|
"performance_schema_stage_classes_lost": "0",
|
||||||
|
"performance_schema_statement_classes_lost": "0",
|
||||||
|
"performance_schema_table_handles_lost": "0",
|
||||||
|
"performance_schema_table_instances_lost": "0",
|
||||||
|
"performance_schema_table_lock_stat_lost": "0",
|
||||||
|
"performance_schema_thread_classes_lost": "0",
|
||||||
|
"performance_schema_thread_instances_lost": "0",
|
||||||
|
"performance_schema_users_lost": "0",
|
||||||
|
"prepared_stmt_count": "0",
|
||||||
|
"qcache_free_blocks": "1",
|
||||||
|
"qcache_free_memory": "1031832",
|
||||||
|
"qcache_hits": "0",
|
||||||
|
"qcache_inserts": "0",
|
||||||
|
"qcache_lowmem_prunes": "0",
|
||||||
|
"qcache_not_cached": "2",
|
||||||
|
"qcache_queries_in_cache": "0",
|
||||||
|
"qcache_total_blocks": "1",
|
||||||
|
"queries": "11",
|
||||||
|
"questions": "9",
|
||||||
|
"rsa_public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApBHbP3hArKih5/R7ffR9\n8ZUpK1VCGmD/7PCjOMQiChcb+KYr7PdgokSw/D95Ua84u36bvRHiB+7a6uGSYHZv\nOMB/h+zP0pSdmucCF5CNRuFgA6D0SHP7lql8O1t4x0dBYGkI0bkUgCxZeeRZshLm\nG6rGMXh79j3WlZP2Ny87TbCcgpbvW0tYxM9/pc1em/8y3+hDd3V0tVSBk36TEuJd\nJRr8Of09H0paSNjPdnvaZYdpXyhZUD0frhLSfnCj52OqFcypeelArR/ntEQffqif\n7CQvVm06mD5a5BlflgIVvnCLSH5pTBcn5tnPACFwbxBMYeRzfmRu+TFqFMBaxEhl\nVQIDAQAB\n-----END PUBLIC KEY-----\n",
|
||||||
|
"select_full_join": "0",
|
||||||
|
"select_full_range_join": "0",
|
||||||
|
"select_range": "0",
|
||||||
|
"select_range_check": "0",
|
||||||
|
"select_scan": "6",
|
||||||
|
"slave_open_temp_tables": "0",
|
||||||
|
"slow_launch_threads": "0",
|
||||||
|
"slow_queries": "0",
|
||||||
|
"sort_merge_passes": "0",
|
||||||
|
"sort_range": "0",
|
||||||
|
"sort_rows": "0",
|
||||||
|
"sort_scan": "0",
|
||||||
|
"ssl_accept_renegotiates": "0",
|
||||||
|
"ssl_accepts": "0",
|
||||||
|
"ssl_callback_cache_hits": "0",
|
||||||
|
"ssl_cipher": "",
|
||||||
|
"ssl_cipher_list": "",
|
||||||
|
"ssl_client_connects": "0",
|
||||||
|
"ssl_connect_renegotiates": "0",
|
||||||
|
"ssl_ctx_verify_depth": "18446744073709551615",
|
||||||
|
"ssl_ctx_verify_mode": "5",
|
||||||
|
"ssl_default_timeout": "0",
|
||||||
|
"ssl_finished_accepts": "0",
|
||||||
|
"ssl_finished_connects": "0",
|
||||||
|
"ssl_server_not_after": "Nov 27 02:39:35 2027 GMT",
|
||||||
|
"ssl_server_not_before": "Nov 29 02:39:35 2017 GMT",
|
||||||
|
"ssl_session_cache_hits": "0",
|
||||||
|
"ssl_session_cache_misses": "0",
|
||||||
|
"ssl_session_cache_mode": "SERVER",
|
||||||
|
"ssl_session_cache_overflows": "0",
|
||||||
|
"ssl_session_cache_size": "128",
|
||||||
|
"ssl_session_cache_timeouts": "0",
|
||||||
|
"ssl_sessions_reused": "0",
|
||||||
|
"ssl_used_session_cache_entries": "0",
|
||||||
|
"ssl_verify_depth": "0",
|
||||||
|
"ssl_verify_mode": "0",
|
||||||
|
"ssl_version": "",
|
||||||
|
"table_locks_immediate": "101",
|
||||||
|
"table_locks_waited": "0",
|
||||||
|
"table_open_cache_hits": "1",
|
||||||
|
"table_open_cache_misses": "1",
|
||||||
|
"table_open_cache_overflows": "0",
|
||||||
|
"tc_log_max_pages_used": "0",
|
||||||
|
"tc_log_page_size": "0",
|
||||||
|
"tc_log_page_waits": "0",
|
||||||
|
"threads_cached": "0",
|
||||||
|
"threads_connected": "1",
|
||||||
|
"threads_created": "1",
|
||||||
|
"threads_running": "1",
|
||||||
|
"uptime": "8",
|
||||||
|
"uptime_since_flush_status": "8"
|
||||||
|
}},
|
||||||
|
"local": null
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,362 @@
|
||||||
|
{
|
||||||
|
"global": {"global": {
|
||||||
|
"aborted_clients": "0",
|
||||||
|
"aborted_connects": "0",
|
||||||
|
"binlog_cache_disk_use": "0",
|
||||||
|
"binlog_cache_use": "0",
|
||||||
|
"binlog_stmt_cache_disk_use": "0",
|
||||||
|
"binlog_stmt_cache_use": "0",
|
||||||
|
"bytes_received": "1227",
|
||||||
|
"bytes_sent": "29713",
|
||||||
|
"com_admin_commands": "0",
|
||||||
|
"com_alter_db": "0",
|
||||||
|
"com_alter_db_upgrade": "0",
|
||||||
|
"com_alter_event": "0",
|
||||||
|
"com_alter_function": "0",
|
||||||
|
"com_alter_instance": "0",
|
||||||
|
"com_alter_procedure": "0",
|
||||||
|
"com_alter_server": "0",
|
||||||
|
"com_alter_table": "0",
|
||||||
|
"com_alter_tablespace": "0",
|
||||||
|
"com_alter_user": "0",
|
||||||
|
"com_analyze": "0",
|
||||||
|
"com_assign_to_keycache": "0",
|
||||||
|
"com_begin": "0",
|
||||||
|
"com_binlog": "0",
|
||||||
|
"com_call_procedure": "0",
|
||||||
|
"com_change_db": "0",
|
||||||
|
"com_change_master": "0",
|
||||||
|
"com_change_repl_filter": "0",
|
||||||
|
"com_check": "0",
|
||||||
|
"com_checksum": "0",
|
||||||
|
"com_commit": "0",
|
||||||
|
"com_create_db": "0",
|
||||||
|
"com_create_event": "0",
|
||||||
|
"com_create_function": "0",
|
||||||
|
"com_create_index": "0",
|
||||||
|
"com_create_procedure": "0",
|
||||||
|
"com_create_server": "0",
|
||||||
|
"com_create_table": "0",
|
||||||
|
"com_create_trigger": "0",
|
||||||
|
"com_create_udf": "0",
|
||||||
|
"com_create_user": "0",
|
||||||
|
"com_create_view": "0",
|
||||||
|
"com_dealloc_sql": "0",
|
||||||
|
"com_delete": "0",
|
||||||
|
"com_delete_multi": "0",
|
||||||
|
"com_do": "0",
|
||||||
|
"com_drop_db": "0",
|
||||||
|
"com_drop_event": "0",
|
||||||
|
"com_drop_function": "0",
|
||||||
|
"com_drop_index": "0",
|
||||||
|
"com_drop_procedure": "0",
|
||||||
|
"com_drop_server": "0",
|
||||||
|
"com_drop_table": "0",
|
||||||
|
"com_drop_trigger": "0",
|
||||||
|
"com_drop_user": "0",
|
||||||
|
"com_drop_view": "0",
|
||||||
|
"com_empty_query": "0",
|
||||||
|
"com_execute_sql": "0",
|
||||||
|
"com_explain_other": "0",
|
||||||
|
"com_flush": "0",
|
||||||
|
"com_get_diagnostics": "0",
|
||||||
|
"com_grant": "0",
|
||||||
|
"com_group_replication_start": "0",
|
||||||
|
"com_group_replication_stop": "0",
|
||||||
|
"com_ha_close": "0",
|
||||||
|
"com_ha_open": "0",
|
||||||
|
"com_ha_read": "0",
|
||||||
|
"com_help": "0",
|
||||||
|
"com_insert": "0",
|
||||||
|
"com_insert_select": "0",
|
||||||
|
"com_install_plugin": "0",
|
||||||
|
"com_kill": "0",
|
||||||
|
"com_load": "0",
|
||||||
|
"com_lock_tables": "0",
|
||||||
|
"com_optimize": "0",
|
||||||
|
"com_preload_keys": "0",
|
||||||
|
"com_prepare_sql": "0",
|
||||||
|
"com_purge": "0",
|
||||||
|
"com_purge_before_date": "0",
|
||||||
|
"com_release_savepoint": "0",
|
||||||
|
"com_rename_table": "0",
|
||||||
|
"com_rename_user": "0",
|
||||||
|
"com_repair": "0",
|
||||||
|
"com_replace": "0",
|
||||||
|
"com_replace_select": "0",
|
||||||
|
"com_reset": "0",
|
||||||
|
"com_resignal": "0",
|
||||||
|
"com_revoke": "0",
|
||||||
|
"com_revoke_all": "0",
|
||||||
|
"com_rollback": "0",
|
||||||
|
"com_rollback_to_savepoint": "0",
|
||||||
|
"com_savepoint": "0",
|
||||||
|
"com_select": "2",
|
||||||
|
"com_set_option": "2",
|
||||||
|
"com_show_binlog_events": "0",
|
||||||
|
"com_show_binlogs": "0",
|
||||||
|
"com_show_charsets": "1",
|
||||||
|
"com_show_collations": "1",
|
||||||
|
"com_show_create_db": "0",
|
||||||
|
"com_show_create_event": "0",
|
||||||
|
"com_show_create_func": "0",
|
||||||
|
"com_show_create_proc": "0",
|
||||||
|
"com_show_create_table": "0",
|
||||||
|
"com_show_create_trigger": "0",
|
||||||
|
"com_show_create_user": "0",
|
||||||
|
"com_show_databases": "0",
|
||||||
|
"com_show_engine_logs": "0",
|
||||||
|
"com_show_engine_mutex": "0",
|
||||||
|
"com_show_engine_status": "0",
|
||||||
|
"com_show_errors": "0",
|
||||||
|
"com_show_events": "0",
|
||||||
|
"com_show_fields": "0",
|
||||||
|
"com_show_function_code": "0",
|
||||||
|
"com_show_function_status": "0",
|
||||||
|
"com_show_grants": "0",
|
||||||
|
"com_show_keys": "0",
|
||||||
|
"com_show_master_status": "0",
|
||||||
|
"com_show_open_tables": "0",
|
||||||
|
"com_show_plugins": "0",
|
||||||
|
"com_show_privileges": "0",
|
||||||
|
"com_show_procedure_code": "0",
|
||||||
|
"com_show_procedure_status": "0",
|
||||||
|
"com_show_processlist": "0",
|
||||||
|
"com_show_profile": "0",
|
||||||
|
"com_show_profiles": "0",
|
||||||
|
"com_show_relaylog_events": "0",
|
||||||
|
"com_show_slave_hosts": "0",
|
||||||
|
"com_show_slave_status": "0",
|
||||||
|
"com_show_status": "1",
|
||||||
|
"com_show_storage_engines": "0",
|
||||||
|
"com_show_table_status": "0",
|
||||||
|
"com_show_tables": "0",
|
||||||
|
"com_show_triggers": "0",
|
||||||
|
"com_show_variables": "2",
|
||||||
|
"com_show_warnings": "0",
|
||||||
|
"com_shutdown": "0",
|
||||||
|
"com_signal": "0",
|
||||||
|
"com_slave_start": "0",
|
||||||
|
"com_slave_stop": "0",
|
||||||
|
"com_stmt_close": "0",
|
||||||
|
"com_stmt_execute": "0",
|
||||||
|
"com_stmt_fetch": "0",
|
||||||
|
"com_stmt_prepare": "0",
|
||||||
|
"com_stmt_reprepare": "0",
|
||||||
|
"com_stmt_reset": "0",
|
||||||
|
"com_stmt_send_long_data": "0",
|
||||||
|
"com_truncate": "0",
|
||||||
|
"com_uninstall_plugin": "0",
|
||||||
|
"com_unlock_tables": "0",
|
||||||
|
"com_update": "0",
|
||||||
|
"com_update_multi": "0",
|
||||||
|
"com_xa_commit": "0",
|
||||||
|
"com_xa_end": "0",
|
||||||
|
"com_xa_prepare": "0",
|
||||||
|
"com_xa_recover": "0",
|
||||||
|
"com_xa_rollback": "0",
|
||||||
|
"com_xa_start": "0",
|
||||||
|
"compression": "OFF",
|
||||||
|
"connection_errors_accept": "0",
|
||||||
|
"connection_errors_internal": "0",
|
||||||
|
"connection_errors_max_connections": "0",
|
||||||
|
"connection_errors_peer_address": "0",
|
||||||
|
"connection_errors_select": "0",
|
||||||
|
"connection_errors_tcpwrap": "0",
|
||||||
|
"connections": "4",
|
||||||
|
"created_tmp_disk_tables": "0",
|
||||||
|
"created_tmp_files": "5",
|
||||||
|
"created_tmp_tables": "4",
|
||||||
|
"delayed_errors": "0",
|
||||||
|
"delayed_insert_threads": "0",
|
||||||
|
"delayed_writes": "0",
|
||||||
|
"flush_commands": "1",
|
||||||
|
"handler_commit": "0",
|
||||||
|
"handler_delete": "0",
|
||||||
|
"handler_discover": "0",
|
||||||
|
"handler_external_lock": "4",
|
||||||
|
"handler_mrr_init": "0",
|
||||||
|
"handler_prepare": "0",
|
||||||
|
"handler_read_first": "0",
|
||||||
|
"handler_read_key": "0",
|
||||||
|
"handler_read_last": "0",
|
||||||
|
"handler_read_next": "0",
|
||||||
|
"handler_read_prev": "0",
|
||||||
|
"handler_read_rnd": "0",
|
||||||
|
"handler_read_rnd_next": "2317",
|
||||||
|
"handler_rollback": "0",
|
||||||
|
"handler_savepoint": "0",
|
||||||
|
"handler_savepoint_rollback": "0",
|
||||||
|
"handler_update": "0",
|
||||||
|
"handler_write": "1287",
|
||||||
|
"innodb_available_undo_logs": "128",
|
||||||
|
"innodb_buffer_pool_bytes_data": "11288576",
|
||||||
|
"innodb_buffer_pool_bytes_dirty": "0",
|
||||||
|
"innodb_buffer_pool_dump_status": "Dumping of buffer pool not started",
|
||||||
|
"innodb_buffer_pool_load_status": "Buffer pool(s) load completed at 171205 23:01:14",
|
||||||
|
"innodb_buffer_pool_pages_data": "689",
|
||||||
|
"innodb_buffer_pool_pages_dirty": "0",
|
||||||
|
"innodb_buffer_pool_pages_flushed": "37",
|
||||||
|
"innodb_buffer_pool_pages_free": "7502",
|
||||||
|
"innodb_buffer_pool_pages_misc": "0",
|
||||||
|
"innodb_buffer_pool_pages_total": "8191",
|
||||||
|
"innodb_buffer_pool_read_ahead": "0",
|
||||||
|
"innodb_buffer_pool_read_ahead_evicted": "0",
|
||||||
|
"innodb_buffer_pool_read_ahead_rnd": "0",
|
||||||
|
"innodb_buffer_pool_read_requests": "1833",
|
||||||
|
"innodb_buffer_pool_reads": "655",
|
||||||
|
"innodb_buffer_pool_resize_status": "",
|
||||||
|
"innodb_buffer_pool_wait_free": "0",
|
||||||
|
"innodb_buffer_pool_write_requests": "515",
|
||||||
|
"innodb_data_fsyncs": "7",
|
||||||
|
"innodb_data_pending_fsyncs": "0",
|
||||||
|
"innodb_data_pending_reads": "0",
|
||||||
|
"innodb_data_pending_writes": "0",
|
||||||
|
"innodb_data_read": "10801664",
|
||||||
|
"innodb_data_reads": "683",
|
||||||
|
"innodb_data_writes": "54",
|
||||||
|
"innodb_data_written": "641024",
|
||||||
|
"innodb_dblwr_pages_written": "2",
|
||||||
|
"innodb_dblwr_writes": "1",
|
||||||
|
"innodb_log_waits": "0",
|
||||||
|
"innodb_log_write_requests": "0",
|
||||||
|
"innodb_log_writes": "2",
|
||||||
|
"innodb_num_open_files": "20",
|
||||||
|
"innodb_os_log_fsyncs": "4",
|
||||||
|
"innodb_os_log_pending_fsyncs": "0",
|
||||||
|
"innodb_os_log_pending_writes": "0",
|
||||||
|
"innodb_os_log_written": "1024",
|
||||||
|
"innodb_page_size": "16384",
|
||||||
|
"innodb_pages_created": "35",
|
||||||
|
"innodb_pages_read": "654",
|
||||||
|
"innodb_pages_written": "37",
|
||||||
|
"innodb_row_lock_current_waits": "0",
|
||||||
|
"innodb_row_lock_time": "0",
|
||||||
|
"innodb_row_lock_time_avg": "0",
|
||||||
|
"innodb_row_lock_time_max": "0",
|
||||||
|
"innodb_row_lock_waits": "0",
|
||||||
|
"innodb_rows_deleted": "0",
|
||||||
|
"innodb_rows_inserted": "0",
|
||||||
|
"innodb_rows_read": "8",
|
||||||
|
"innodb_rows_updated": "0",
|
||||||
|
"innodb_truncated_status_writes": "0",
|
||||||
|
"key_blocks_not_flushed": "0",
|
||||||
|
"key_blocks_unused": "6695",
|
||||||
|
"key_blocks_used": "3",
|
||||||
|
"key_read_requests": "6",
|
||||||
|
"key_reads": "3",
|
||||||
|
"key_write_requests": "0",
|
||||||
|
"key_writes": "0",
|
||||||
|
"last_query_cost": "0.000000",
|
||||||
|
"last_query_partial_plans": "2",
|
||||||
|
"locked_connects": "0",
|
||||||
|
"max_execution_time_exceeded": "0",
|
||||||
|
"max_execution_time_set": "0",
|
||||||
|
"max_execution_time_set_failed": "0",
|
||||||
|
"max_used_connections": "1",
|
||||||
|
"max_used_connections_time": "2017-12-05 23:01:20",
|
||||||
|
"not_flushed_delayed_rows": "0",
|
||||||
|
"ongoing_anonymous_transaction_count": "0",
|
||||||
|
"open_files": "14",
|
||||||
|
"open_streams": "0",
|
||||||
|
"open_table_definitions": "228",
|
||||||
|
"open_tables": "100",
|
||||||
|
"opened_files": "357",
|
||||||
|
"opened_table_definitions": "0",
|
||||||
|
"opened_tables": "1",
|
||||||
|
"performance_schema_accounts_lost": "0",
|
||||||
|
"performance_schema_cond_classes_lost": "0",
|
||||||
|
"performance_schema_cond_instances_lost": "0",
|
||||||
|
"performance_schema_digest_lost": "0",
|
||||||
|
"performance_schema_file_classes_lost": "0",
|
||||||
|
"performance_schema_file_handles_lost": "0",
|
||||||
|
"performance_schema_file_instances_lost": "0",
|
||||||
|
"performance_schema_hosts_lost": "0",
|
||||||
|
"performance_schema_index_stat_lost": "0",
|
||||||
|
"performance_schema_locker_lost": "0",
|
||||||
|
"performance_schema_memory_classes_lost": "0",
|
||||||
|
"performance_schema_metadata_lock_lost": "0",
|
||||||
|
"performance_schema_mutex_classes_lost": "0",
|
||||||
|
"performance_schema_mutex_instances_lost": "0",
|
||||||
|
"performance_schema_nested_statement_lost": "0",
|
||||||
|
"performance_schema_prepared_statements_lost": "0",
|
||||||
|
"performance_schema_program_lost": "0",
|
||||||
|
"performance_schema_rwlock_classes_lost": "0",
|
||||||
|
"performance_schema_rwlock_instances_lost": "0",
|
||||||
|
"performance_schema_session_connect_attrs_lost": "0",
|
||||||
|
"performance_schema_socket_classes_lost": "0",
|
||||||
|
"performance_schema_socket_instances_lost": "0",
|
||||||
|
"performance_schema_stage_classes_lost": "0",
|
||||||
|
"performance_schema_statement_classes_lost": "0",
|
||||||
|
"performance_schema_table_handles_lost": "0",
|
||||||
|
"performance_schema_table_instances_lost": "0",
|
||||||
|
"performance_schema_table_lock_stat_lost": "0",
|
||||||
|
"performance_schema_thread_classes_lost": "0",
|
||||||
|
"performance_schema_thread_instances_lost": "0",
|
||||||
|
"performance_schema_users_lost": "0",
|
||||||
|
"prepared_stmt_count": "0",
|
||||||
|
"qcache_free_blocks": "1",
|
||||||
|
"qcache_free_memory": "1031832",
|
||||||
|
"qcache_hits": "0",
|
||||||
|
"qcache_inserts": "0",
|
||||||
|
"qcache_lowmem_prunes": "0",
|
||||||
|
"qcache_not_cached": "2",
|
||||||
|
"qcache_queries_in_cache": "0",
|
||||||
|
"qcache_total_blocks": "1",
|
||||||
|
"queries": "11",
|
||||||
|
"questions": "9",
|
||||||
|
"rsa_public_key": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApBHbP3hArKih5/R7ffR9\n8ZUpK1VCGmD/7PCjOMQiChcb+KYr7PdgokSw/D95Ua84u36bvRHiB+7a6uGSYHZv\nOMB/h+zP0pSdmucCF5CNRuFgA6D0SHP7lql8O1t4x0dBYGkI0bkUgCxZeeRZshLm\nG6rGMXh79j3WlZP2Ny87TbCcgpbvW0tYxM9/pc1em/8y3+hDd3V0tVSBk36TEuJd\nJRr8Of09H0paSNjPdnvaZYdpXyhZUD0frhLSfnCj52OqFcypeelArR/ntEQffqif\n7CQvVm06mD5a5BlflgIVvnCLSH5pTBcn5tnPACFwbxBMYeRzfmRu+TFqFMBaxEhl\nVQIDAQAB\n-----END PUBLIC KEY-----\n",
|
||||||
|
"select_full_join": "0",
|
||||||
|
"select_full_range_join": "0",
|
||||||
|
"select_range": "0",
|
||||||
|
"select_range_check": "0",
|
||||||
|
"select_scan": "6",
|
||||||
|
"slave_open_temp_tables": "0",
|
||||||
|
"slow_launch_threads": "0",
|
||||||
|
"slow_queries": "0",
|
||||||
|
"sort_merge_passes": "0",
|
||||||
|
"sort_range": "0",
|
||||||
|
"sort_rows": "0",
|
||||||
|
"sort_scan": "0",
|
||||||
|
"ssl_accept_renegotiates": "0",
|
||||||
|
"ssl_accepts": "0",
|
||||||
|
"ssl_callback_cache_hits": "0",
|
||||||
|
"ssl_cipher": "",
|
||||||
|
"ssl_cipher_list": "",
|
||||||
|
"ssl_client_connects": "0",
|
||||||
|
"ssl_connect_renegotiates": "0",
|
||||||
|
"ssl_ctx_verify_depth": "18446744073709551615",
|
||||||
|
"ssl_ctx_verify_mode": "5",
|
||||||
|
"ssl_default_timeout": "0",
|
||||||
|
"ssl_finished_accepts": "0",
|
||||||
|
"ssl_finished_connects": "0",
|
||||||
|
"ssl_server_not_after": "Nov 27 02:39:35 2027 GMT",
|
||||||
|
"ssl_server_not_before": "Nov 29 02:39:35 2017 GMT",
|
||||||
|
"ssl_session_cache_hits": "0",
|
||||||
|
"ssl_session_cache_misses": "0",
|
||||||
|
"ssl_session_cache_mode": "SERVER",
|
||||||
|
"ssl_session_cache_overflows": "0",
|
||||||
|
"ssl_session_cache_size": "128",
|
||||||
|
"ssl_session_cache_timeouts": "0",
|
||||||
|
"ssl_sessions_reused": "0",
|
||||||
|
"ssl_used_session_cache_entries": "0",
|
||||||
|
"ssl_verify_depth": "0",
|
||||||
|
"ssl_verify_mode": "0",
|
||||||
|
"ssl_version": "",
|
||||||
|
"table_locks_immediate": "101",
|
||||||
|
"table_locks_waited": "0",
|
||||||
|
"table_open_cache_hits": "1",
|
||||||
|
"table_open_cache_misses": "1",
|
||||||
|
"table_open_cache_overflows": "0",
|
||||||
|
"tc_log_max_pages_used": "0",
|
||||||
|
"tc_log_page_size": "0",
|
||||||
|
"tc_log_page_waits": "0",
|
||||||
|
"threads_cached": "0",
|
||||||
|
"threads_connected": "1",
|
||||||
|
"threads_created": "1",
|
||||||
|
"threads_running": "1",
|
||||||
|
"uptime": "8",
|
||||||
|
"uptime_since_flush_status": "8"
|
||||||
|
}},
|
||||||
|
"local": null
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"start_time": 1535653369274,
|
||||||
|
"end_time": 1535653559607,
|
||||||
|
"observation_time": 190,
|
||||||
|
"database_type": "mysql",
|
||||||
|
"database_version": "5.7.20",
|
||||||
|
"workload_name": "workload_name"
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"start_time": 1563264802685,
|
||||||
|
"end_time": 1563265002918,
|
||||||
|
"observation_time": 200,
|
||||||
|
"database_type": "oracle",
|
||||||
|
"database_version": "19.0.0.0.0",
|
||||||
|
"workload_name": "tpcc"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,274 @@
|
||||||
|
{
|
||||||
|
"global": {"global": {
|
||||||
|
"DateStyle": "ISO, MDY",
|
||||||
|
"IntervalStyle": "postgres",
|
||||||
|
"TimeZone": "America/New_York",
|
||||||
|
"allow_system_table_mods": "off",
|
||||||
|
"application_name": "",
|
||||||
|
"archive_command": "(disabled)",
|
||||||
|
"archive_mode": "off",
|
||||||
|
"archive_timeout": "0",
|
||||||
|
"array_nulls": "on",
|
||||||
|
"authentication_timeout": "1min",
|
||||||
|
"autovacuum": "on",
|
||||||
|
"autovacuum_analyze_scale_factor": "0.1",
|
||||||
|
"autovacuum_analyze_threshold": "50",
|
||||||
|
"autovacuum_freeze_max_age": "200000000",
|
||||||
|
"autovacuum_max_workers": "3",
|
||||||
|
"autovacuum_multixact_freeze_max_age": "400000000",
|
||||||
|
"autovacuum_naptime": "1min",
|
||||||
|
"autovacuum_vacuum_cost_delay": "20ms",
|
||||||
|
"autovacuum_vacuum_cost_limit": "-1",
|
||||||
|
"autovacuum_vacuum_scale_factor": "0.2",
|
||||||
|
"autovacuum_vacuum_threshold": "50",
|
||||||
|
"autovacuum_work_mem": "-1",
|
||||||
|
"backend_flush_after": "0",
|
||||||
|
"backslash_quote": "safe_encoding",
|
||||||
|
"bgwriter_delay": "200ms",
|
||||||
|
"bgwriter_flush_after": "0",
|
||||||
|
"bgwriter_lru_maxpages": "100",
|
||||||
|
"bgwriter_lru_multiplier": "2",
|
||||||
|
"block_size": "8192",
|
||||||
|
"bonjour": "off",
|
||||||
|
"bonjour_name": "",
|
||||||
|
"bytea_output": "hex",
|
||||||
|
"check_function_bodies": "on",
|
||||||
|
"checkpoint_completion_target": "0.5",
|
||||||
|
"checkpoint_flush_after": "0",
|
||||||
|
"checkpoint_timeout": "5min",
|
||||||
|
"checkpoint_warning": "30s",
|
||||||
|
"client_encoding": "UTF8",
|
||||||
|
"client_min_messages": "notice",
|
||||||
|
"cluster_name": "",
|
||||||
|
"commit_delay": "0",
|
||||||
|
"commit_siblings": "5",
|
||||||
|
"config_file": "/Users/MacadamiaKitten/Desktop/psql_db/postgresql.conf",
|
||||||
|
"constraint_exclusion": "partition",
|
||||||
|
"cpu_index_tuple_cost": "0.005",
|
||||||
|
"cpu_operator_cost": "0.0025",
|
||||||
|
"cpu_tuple_cost": "0.01",
|
||||||
|
"cursor_tuple_fraction": "0.1",
|
||||||
|
"data_checksums": "off",
|
||||||
|
"data_directory": "/Users/MacadamiaKitten/Desktop/psql_db",
|
||||||
|
"db_user_namespace": "off",
|
||||||
|
"deadlock_timeout": "1s",
|
||||||
|
"debug_assertions": "off",
|
||||||
|
"debug_pretty_print": "on",
|
||||||
|
"debug_print_parse": "off",
|
||||||
|
"debug_print_plan": "off",
|
||||||
|
"debug_print_rewritten": "off",
|
||||||
|
"default_statistics_target": "100",
|
||||||
|
"default_tablespace": "",
|
||||||
|
"default_text_search_config": "pg_catalog.english",
|
||||||
|
"default_transaction_deferrable": "off",
|
||||||
|
"default_transaction_isolation": "read committed",
|
||||||
|
"default_transaction_read_only": "off",
|
||||||
|
"default_with_oids": "off",
|
||||||
|
"dynamic_library_path": "$libdir",
|
||||||
|
"dynamic_shared_memory_type": "posix",
|
||||||
|
"effective_cache_size": "4GB",
|
||||||
|
"effective_io_concurrency": "0",
|
||||||
|
"enable_bitmapscan": "on",
|
||||||
|
"enable_gathermerge": "on",
|
||||||
|
"enable_hashagg": "on",
|
||||||
|
"enable_hashjoin": "on",
|
||||||
|
"enable_indexonlyscan": "on",
|
||||||
|
"enable_indexscan": "on",
|
||||||
|
"enable_material": "on",
|
||||||
|
"enable_mergejoin": "on",
|
||||||
|
"enable_nestloop": "on",
|
||||||
|
"enable_seqscan": "on",
|
||||||
|
"enable_sort": "on",
|
||||||
|
"enable_tidscan": "on",
|
||||||
|
"escape_string_warning": "on",
|
||||||
|
"event_source": "PostgreSQL",
|
||||||
|
"exit_on_error": "off",
|
||||||
|
"external_pid_file": "",
|
||||||
|
"extra_float_digits": "3",
|
||||||
|
"force_parallel_mode": "off",
|
||||||
|
"from_collapse_limit": "8",
|
||||||
|
"fsync": "on",
|
||||||
|
"full_page_writes": "on",
|
||||||
|
"geqo": "on",
|
||||||
|
"geqo_effort": "5",
|
||||||
|
"geqo_generations": "0",
|
||||||
|
"geqo_pool_size": "0",
|
||||||
|
"geqo_seed": "0",
|
||||||
|
"geqo_selection_bias": "2",
|
||||||
|
"geqo_threshold": "12",
|
||||||
|
"gin_fuzzy_search_limit": "0",
|
||||||
|
"gin_pending_list_limit": "4MB",
|
||||||
|
"hba_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_hba.conf",
|
||||||
|
"hot_standby": "on",
|
||||||
|
"hot_standby_feedback": "off",
|
||||||
|
"huge_pages": "try",
|
||||||
|
"ident_file": "/Users/MacadamiaKitten/Desktop/psql_db/pg_ident.conf",
|
||||||
|
"idle_in_transaction_session_timeout": "0",
|
||||||
|
"ignore_checksum_failure": "off",
|
||||||
|
"ignore_system_indexes": "off",
|
||||||
|
"integer_datetimes": "on",
|
||||||
|
"join_collapse_limit": "8",
|
||||||
|
"krb_caseins_users": "off",
|
||||||
|
"krb_server_keyfile": "FILE:/usr/local/etc/postgresql/krb5.keytab",
|
||||||
|
"lc_collate": "en_US.UTF-8",
|
||||||
|
"lc_ctype": "en_US.UTF-8",
|
||||||
|
"lc_messages": "en_US.UTF-8",
|
||||||
|
"lc_monetary": "en_US.UTF-8",
|
||||||
|
"lc_numeric": "en_US.UTF-8",
|
||||||
|
"lc_time": "en_US.UTF-8",
|
||||||
|
"listen_addresses": "localhost",
|
||||||
|
"lo_compat_privileges": "off",
|
||||||
|
"local_preload_libraries": "",
|
||||||
|
"lock_timeout": "0",
|
||||||
|
"log_autovacuum_min_duration": "-1",
|
||||||
|
"log_checkpoints": "off",
|
||||||
|
"log_connections": "off",
|
||||||
|
"log_destination": "stderr",
|
||||||
|
"log_directory": "log",
|
||||||
|
"log_disconnections": "off",
|
||||||
|
"log_duration": "off",
|
||||||
|
"log_error_verbosity": "default",
|
||||||
|
"log_executor_stats": "off",
|
||||||
|
"log_file_mode": "0600",
|
||||||
|
"log_filename": "postgresql-%Y-%m-%d_%H%M%S.log",
|
||||||
|
"log_hostname": "off",
|
||||||
|
"log_line_prefix": "%m [%p] ",
|
||||||
|
"log_lock_waits": "off",
|
||||||
|
"log_min_duration_statement": "-1",
|
||||||
|
"log_min_error_statement": "error",
|
||||||
|
"log_min_messages": "warning",
|
||||||
|
"log_parser_stats": "off",
|
||||||
|
"log_planner_stats": "off",
|
||||||
|
"log_replication_commands": "off",
|
||||||
|
"log_rotation_age": "1d",
|
||||||
|
"log_rotation_size": "10MB",
|
||||||
|
"log_statement": "none",
|
||||||
|
"log_statement_stats": "off",
|
||||||
|
"log_temp_files": "-1",
|
||||||
|
"log_timezone": "US/Eastern",
|
||||||
|
"log_truncate_on_rotation": "off",
|
||||||
|
"logging_collector": "off",
|
||||||
|
"maintenance_work_mem": "64MB",
|
||||||
|
"max_connections": "100",
|
||||||
|
"max_files_per_process": "1000",
|
||||||
|
"max_function_args": "100",
|
||||||
|
"max_identifier_length": "63",
|
||||||
|
"max_index_keys": "32",
|
||||||
|
"max_locks_per_transaction": "64",
|
||||||
|
"max_logical_replication_workers": "4",
|
||||||
|
"max_parallel_workers": "8",
|
||||||
|
"max_parallel_workers_per_gather": "2",
|
||||||
|
"max_pred_locks_per_page": "2",
|
||||||
|
"max_pred_locks_per_relation": "-2",
|
||||||
|
"max_pred_locks_per_transaction": "64",
|
||||||
|
"max_prepared_transactions": "0",
|
||||||
|
"max_replication_slots": "10",
|
||||||
|
"max_stack_depth": "2MB",
|
||||||
|
"max_standby_archive_delay": "30s",
|
||||||
|
"max_standby_streaming_delay": "30s",
|
||||||
|
"max_sync_workers_per_subscription": "2",
|
||||||
|
"max_wal_senders": "10",
|
||||||
|
"max_wal_size": "1GB",
|
||||||
|
"max_worker_processes": "8",
|
||||||
|
"min_parallel_index_scan_size": "512kB",
|
||||||
|
"min_parallel_table_scan_size": "8MB",
|
||||||
|
"min_wal_size": "80MB",
|
||||||
|
"old_snapshot_threshold": "-1",
|
||||||
|
"operator_precedence_warning": "off",
|
||||||
|
"parallel_setup_cost": "1000",
|
||||||
|
"parallel_tuple_cost": "0.1",
|
||||||
|
"password_encryption": "md5",
|
||||||
|
"port": "5432",
|
||||||
|
"post_auth_delay": "0",
|
||||||
|
"pre_auth_delay": "0",
|
||||||
|
"quote_all_identifiers": "off",
|
||||||
|
"random_page_cost": "4",
|
||||||
|
"replacement_sort_tuples": "150000",
|
||||||
|
"restart_after_crash": "on",
|
||||||
|
"row_security": "on",
|
||||||
|
"search_path": "\"$user\", public",
|
||||||
|
"segment_size": "1GB",
|
||||||
|
"seq_page_cost": "1",
|
||||||
|
"server_encoding": "UTF8",
|
||||||
|
"server_version": "10.1",
|
||||||
|
"server_version_num": "100001",
|
||||||
|
"session_preload_libraries": "",
|
||||||
|
"session_replication_role": "origin",
|
||||||
|
"shared_buffers": "128MB",
|
||||||
|
"shared_preload_libraries": "",
|
||||||
|
"ssl": "off",
|
||||||
|
"ssl_ca_file": "",
|
||||||
|
"ssl_cert_file": "server.crt",
|
||||||
|
"ssl_ciphers": "HIGH:MEDIUM:+3DES:!aNULL",
|
||||||
|
"ssl_crl_file": "",
|
||||||
|
"ssl_dh_params_file": "",
|
||||||
|
"ssl_ecdh_curve": "prime256v1",
|
||||||
|
"ssl_key_file": "server.key",
|
||||||
|
"ssl_prefer_server_ciphers": "on",
|
||||||
|
"standard_conforming_strings": "on",
|
||||||
|
"statement_timeout": "0",
|
||||||
|
"stats_temp_directory": "pg_stat_tmp",
|
||||||
|
"superuser_reserved_connections": "3",
|
||||||
|
"synchronize_seqscans": "on",
|
||||||
|
"synchronous_commit": "on",
|
||||||
|
"synchronous_standby_names": "",
|
||||||
|
"syslog_facility": "local0",
|
||||||
|
"syslog_ident": "postgres",
|
||||||
|
"syslog_sequence_numbers": "on",
|
||||||
|
"syslog_split_messages": "on",
|
||||||
|
"tcp_keepalives_count": "8",
|
||||||
|
"tcp_keepalives_idle": "7200",
|
||||||
|
"tcp_keepalives_interval": "75",
|
||||||
|
"temp_buffers": "8MB",
|
||||||
|
"temp_file_limit": "-1",
|
||||||
|
"temp_tablespaces": "",
|
||||||
|
"timezone_abbreviations": "Default",
|
||||||
|
"trace_notify": "off",
|
||||||
|
"trace_recovery_messages": "log",
|
||||||
|
"trace_sort": "off",
|
||||||
|
"track_activities": "on",
|
||||||
|
"track_activity_query_size": "1024",
|
||||||
|
"track_commit_timestamp": "off",
|
||||||
|
"track_counts": "on",
|
||||||
|
"track_functions": "none",
|
||||||
|
"track_io_timing": "off",
|
||||||
|
"transaction_deferrable": "off",
|
||||||
|
"transaction_isolation": "read committed",
|
||||||
|
"transaction_read_only": "off",
|
||||||
|
"transform_null_equals": "off",
|
||||||
|
"unix_socket_directories": "/tmp",
|
||||||
|
"unix_socket_group": "",
|
||||||
|
"unix_socket_permissions": "0777",
|
||||||
|
"update_process_title": "on",
|
||||||
|
"vacuum_cost_delay": "0",
|
||||||
|
"vacuum_cost_limit": "200",
|
||||||
|
"vacuum_cost_page_dirty": "20",
|
||||||
|
"vacuum_cost_page_hit": "1",
|
||||||
|
"vacuum_cost_page_miss": "10",
|
||||||
|
"vacuum_defer_cleanup_age": "0",
|
||||||
|
"vacuum_freeze_min_age": "50000000",
|
||||||
|
"vacuum_freeze_table_age": "150000000",
|
||||||
|
"vacuum_multixact_freeze_min_age": "5000000",
|
||||||
|
"vacuum_multixact_freeze_table_age": "150000000",
|
||||||
|
"wal_block_size": "8192",
|
||||||
|
"wal_buffers": "4MB",
|
||||||
|
"wal_compression": "off",
|
||||||
|
"wal_consistency_checking": "",
|
||||||
|
"wal_keep_segments": "0",
|
||||||
|
"wal_level": "replica",
|
||||||
|
"wal_log_hints": "off",
|
||||||
|
"wal_receiver_status_interval": "10s",
|
||||||
|
"wal_receiver_timeout": "1min",
|
||||||
|
"wal_retrieve_retry_interval": "5s",
|
||||||
|
"wal_segment_size": "16MB",
|
||||||
|
"wal_sender_timeout": "1min",
|
||||||
|
"wal_sync_method": "open_datasync",
|
||||||
|
"wal_writer_delay": "200ms",
|
||||||
|
"wal_writer_flush_after": "1MB",
|
||||||
|
"work_mem": "4MB",
|
||||||
|
"xmlbinary": "base64",
|
||||||
|
"xmloption": "content",
|
||||||
|
"zero_damaged_pages": "off"
|
||||||
|
}},
|
||||||
|
"local": null
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,582 @@
|
||||||
|
{
|
||||||
|
"global": {
|
||||||
|
"pg_stat_archiver": {
|
||||||
|
"archived_count": "0",
|
||||||
|
"failed_count": "0",
|
||||||
|
"stats_reset": "2017-11-10 10:59:47.397075-05"
|
||||||
|
},
|
||||||
|
"pg_stat_bgwriter": {
|
||||||
|
"buffers_alloc": "87670",
|
||||||
|
"buffers_backend": "81032",
|
||||||
|
"buffers_backend_fsync": "0",
|
||||||
|
"buffers_checkpoint": "33250",
|
||||||
|
"buffers_clean": "49590",
|
||||||
|
"checkpoint_sync_time": "19",
|
||||||
|
"checkpoint_write_time": "597851",
|
||||||
|
"checkpoints_req": "2",
|
||||||
|
"checkpoints_timed": "1277",
|
||||||
|
"maxwritten_clean": "325",
|
||||||
|
"stats_reset": "2017-11-10 10:59:47.397075-05"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"local": {
|
||||||
|
"table": {
|
||||||
|
"pg_stat_user_tables": {
|
||||||
|
"history": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "60854",
|
||||||
|
"n_mod_since_analyze": "854",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "60854",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16536",
|
||||||
|
"relname": "history",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "2",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "2",
|
||||||
|
"autovacuum_count": "2",
|
||||||
|
"idx_scan": "202634",
|
||||||
|
"idx_tup_fetch": "202634",
|
||||||
|
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
|
||||||
|
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "2",
|
||||||
|
"n_mod_since_analyze": "0",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "854",
|
||||||
|
"n_tup_ins": "2",
|
||||||
|
"n_tup_upd": "854",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "644561",
|
||||||
|
"idx_tup_fetch": "644561",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
|
||||||
|
"n_dead_tup": "4364",
|
||||||
|
"n_live_tup": "200000",
|
||||||
|
"n_mod_since_analyze": "8901",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "5305",
|
||||||
|
"n_tup_ins": "200000",
|
||||||
|
"n_tup_upd": "8901",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "125261",
|
||||||
|
"idx_tup_fetch": "85299628",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
|
||||||
|
"n_dead_tup": "1510",
|
||||||
|
"n_live_tup": "60000",
|
||||||
|
"n_mod_since_analyze": "1594",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "262",
|
||||||
|
"n_tup_ins": "60000",
|
||||||
|
"n_tup_upd": "1594",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"order_line": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "1655",
|
||||||
|
"idx_tup_fetch": "33762",
|
||||||
|
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
|
||||||
|
"n_dead_tup": "2550",
|
||||||
|
"n_live_tup": "608373",
|
||||||
|
"n_mod_since_analyze": "16230",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "5393",
|
||||||
|
"n_tup_ins": "608373",
|
||||||
|
"n_tup_upd": "7329",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "627652",
|
||||||
|
"idx_tup_fetch": "627652",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
|
||||||
|
"n_dead_tup": "117",
|
||||||
|
"n_live_tup": "60889",
|
||||||
|
"n_mod_since_analyze": "1629",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "662",
|
||||||
|
"n_tup_ins": "60900",
|
||||||
|
"n_tup_upd": "740",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "4",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "1481",
|
||||||
|
"idx_tup_fetch": "1480",
|
||||||
|
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
|
||||||
|
"n_dead_tup": "751",
|
||||||
|
"n_live_tup": "16964",
|
||||||
|
"n_mod_since_analyze": "1629",
|
||||||
|
"n_tup_del": "740",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "17715",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "2",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "122234",
|
||||||
|
"idx_tup_fetch": "122234",
|
||||||
|
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
|
||||||
|
"n_dead_tup": "33",
|
||||||
|
"n_live_tup": "20",
|
||||||
|
"n_mod_since_analyze": "0",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "1754",
|
||||||
|
"n_tup_ins": "20",
|
||||||
|
"n_tup_upd": "1754",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "2221",
|
||||||
|
"seq_tup_read": "41522",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "209020",
|
||||||
|
"idx_tup_fetch": "209009",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "102000",
|
||||||
|
"n_mod_since_analyze": "2000",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "100000",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_statio_user_tables": {
|
||||||
|
"history": {
|
||||||
|
"heap_blks_hit": "184380",
|
||||||
|
"heap_blks_read": "746",
|
||||||
|
"relid": "16536",
|
||||||
|
"relname": "history",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"order_line": {
|
||||||
|
"heap_blks_hit": "1869417",
|
||||||
|
"heap_blks_read": "12419",
|
||||||
|
"idx_blks_hit": "1788651",
|
||||||
|
"idx_blks_read": "3708",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"heap_blks_hit": "404486",
|
||||||
|
"heap_blks_read": "80",
|
||||||
|
"idx_blks_hit": "202643",
|
||||||
|
"idx_blks_read": "6",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"heap_blks_hit": "37856",
|
||||||
|
"heap_blks_read": "192",
|
||||||
|
"idx_blks_hit": "38225",
|
||||||
|
"idx_blks_read": "134",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"heap_blks_hit": "1920817",
|
||||||
|
"heap_blks_read": "11757",
|
||||||
|
"idx_blks_hit": "2447522",
|
||||||
|
"idx_blks_read": "1530",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"heap_blks_hit": "1378399",
|
||||||
|
"heap_blks_read": "928",
|
||||||
|
"idx_blks_hit": "3979052",
|
||||||
|
"idx_blks_read": "1881",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"heap_blks_hit": "249754",
|
||||||
|
"heap_blks_read": "3",
|
||||||
|
"idx_blks_hit": "122259",
|
||||||
|
"idx_blks_read": "5",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"heap_blks_hit": "509702",
|
||||||
|
"heap_blks_read": "4542",
|
||||||
|
"idx_blks_hit": "617914",
|
||||||
|
"idx_blks_read": "877",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"heap_blks_hit": "70136669",
|
||||||
|
"heap_blks_read": "13826",
|
||||||
|
"idx_blks_hit": "1411491",
|
||||||
|
"idx_blks_read": "2716",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public",
|
||||||
|
"tidx_blks_hit": "0",
|
||||||
|
"tidx_blks_read": "0",
|
||||||
|
"toast_blks_hit": "0",
|
||||||
|
"toast_blks_read": "0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"database": {
|
||||||
|
"pg_stat_database": {
|
||||||
|
"postgres": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "115229324",
|
||||||
|
"blks_read": "104188",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "12558",
|
||||||
|
"datname": "postgres",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "1",
|
||||||
|
"stats_reset": "2017-11-10 11:14:57.116228-05",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "1818",
|
||||||
|
"tup_fetched": "103355344",
|
||||||
|
"tup_inserted": "2210752",
|
||||||
|
"tup_returned": "110741743",
|
||||||
|
"tup_updated": "32675",
|
||||||
|
"xact_commit": "19082",
|
||||||
|
"xact_rollback": "17"
|
||||||
|
},
|
||||||
|
"tpcc": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "16384",
|
||||||
|
"datname": "tpcc",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
},
|
||||||
|
"template1": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "1",
|
||||||
|
"datname": "template1",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
},
|
||||||
|
"template0": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "12557",
|
||||||
|
"datname": "template0",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_stat_database_conflicts": {
|
||||||
|
"postgres": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "12558",
|
||||||
|
"datname": "postgres"
|
||||||
|
},
|
||||||
|
"tpcc": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "16384",
|
||||||
|
"datname": "tpcc"
|
||||||
|
},
|
||||||
|
"template1": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "1",
|
||||||
|
"datname": "template1"
|
||||||
|
},
|
||||||
|
"template0": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "12557",
|
||||||
|
"datname": "template0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"pg_stat_user_indexes": {
|
||||||
|
"order_line": {
|
||||||
|
"idx_scan": "1655",
|
||||||
|
"idx_tup_fetch": "33762",
|
||||||
|
"idx_tup_read": "35698",
|
||||||
|
"indexrelid": "16516",
|
||||||
|
"indexrelname": "order_line_pkey",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"idx_scan": "1481",
|
||||||
|
"idx_tup_fetch": "1480",
|
||||||
|
"idx_tup_read": "2200",
|
||||||
|
"indexrelid": "16521",
|
||||||
|
"indexrelname": "new_order_pkey",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"idx_scan": "644561",
|
||||||
|
"idx_tup_fetch": "644561",
|
||||||
|
"idx_tup_read": "647319",
|
||||||
|
"indexrelid": "16526",
|
||||||
|
"indexrelname": "stock_pkey",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"idx_scan": "616371",
|
||||||
|
"idx_tup_fetch": "616371",
|
||||||
|
"idx_tup_read": "616371",
|
||||||
|
"indexrelid": "16565",
|
||||||
|
"indexrelname": "idx_order",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"idx_scan": "82442",
|
||||||
|
"idx_tup_fetch": "85256809",
|
||||||
|
"idx_tup_read": "85256841",
|
||||||
|
"indexrelid": "16564",
|
||||||
|
"indexrelname": "idx_customer_name",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"idx_scan": "122234",
|
||||||
|
"idx_tup_fetch": "122234",
|
||||||
|
"idx_tup_read": "122234",
|
||||||
|
"indexrelid": "16552",
|
||||||
|
"indexrelname": "district_pkey",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"idx_scan": "209020",
|
||||||
|
"idx_tup_fetch": "209009",
|
||||||
|
"idx_tup_read": "209009",
|
||||||
|
"indexrelid": "16557",
|
||||||
|
"indexrelname": "item_pkey",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"idx_scan": "202634",
|
||||||
|
"idx_tup_fetch": "201331",
|
||||||
|
"idx_tup_read": "202634",
|
||||||
|
"indexrelid": "16562",
|
||||||
|
"indexrelname": "warehouse_pkey",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_statio_user_indexes": {
|
||||||
|
"order_line": {
|
||||||
|
"idx_blks_hit": "1788651",
|
||||||
|
"idx_blks_read": "3708",
|
||||||
|
"indexrelid": "16516",
|
||||||
|
"indexrelname": "order_line_pkey",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"idx_blks_hit": "38225",
|
||||||
|
"idx_blks_read": "134",
|
||||||
|
"indexrelid": "16521",
|
||||||
|
"indexrelname": "new_order_pkey",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"idx_blks_hit": "2447522",
|
||||||
|
"idx_blks_read": "1530",
|
||||||
|
"indexrelid": "16526",
|
||||||
|
"indexrelname": "stock_pkey",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"idx_blks_hit": "3689479",
|
||||||
|
"idx_blks_read": "733",
|
||||||
|
"indexrelid": "16565",
|
||||||
|
"indexrelname": "idx_order",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"idx_blks_hit": "1151523",
|
||||||
|
"idx_blks_read": "1589",
|
||||||
|
"indexrelid": "16564",
|
||||||
|
"indexrelname": "idx_customer_name",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"idx_blks_hit": "122259",
|
||||||
|
"idx_blks_read": "5",
|
||||||
|
"indexrelid": "16552",
|
||||||
|
"indexrelname": "district_pkey",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"idx_blks_hit": "617914",
|
||||||
|
"idx_blks_read": "877",
|
||||||
|
"indexrelid": "16557",
|
||||||
|
"indexrelname": "item_pkey",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"idx_blks_hit": "202643",
|
||||||
|
"idx_blks_read": "6",
|
||||||
|
"indexrelid": "16562",
|
||||||
|
"indexrelname": "warehouse_pkey",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,582 @@
|
||||||
|
{
|
||||||
|
"global": {
|
||||||
|
"pg_stat_archiver": {
|
||||||
|
"archived_count": "0",
|
||||||
|
"failed_count": "0",
|
||||||
|
"stats_reset": "2017-11-10 10:59:47.397075-05"
|
||||||
|
},
|
||||||
|
"pg_stat_bgwriter": {
|
||||||
|
"buffers_alloc": "87670",
|
||||||
|
"buffers_backend": "81032",
|
||||||
|
"buffers_backend_fsync": "0",
|
||||||
|
"buffers_checkpoint": "33250",
|
||||||
|
"buffers_clean": "49590",
|
||||||
|
"checkpoint_sync_time": "19",
|
||||||
|
"checkpoint_write_time": "597851",
|
||||||
|
"checkpoints_req": "2",
|
||||||
|
"checkpoints_timed": "1277",
|
||||||
|
"maxwritten_clean": "325",
|
||||||
|
"stats_reset": "2017-11-10 10:59:47.397075-05"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"local": {
|
||||||
|
"table": {
|
||||||
|
"pg_stat_user_tables": {
|
||||||
|
"history": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:02.567618-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "60854",
|
||||||
|
"n_mod_since_analyze": "854",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "60854",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16536",
|
||||||
|
"relname": "history",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "2",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "2",
|
||||||
|
"autovacuum_count": "2",
|
||||||
|
"idx_scan": "202634",
|
||||||
|
"idx_tup_fetch": "202634",
|
||||||
|
"last_autoanalyze": "2017-11-20 19:23:34.236294-05",
|
||||||
|
"last_autovacuum": "2017-11-20 19:23:34.235793-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "2",
|
||||||
|
"n_mod_since_analyze": "0",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "854",
|
||||||
|
"n_tup_ins": "2",
|
||||||
|
"n_tup_upd": "854",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "644561",
|
||||||
|
"idx_tup_fetch": "644561",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:01.368483-05",
|
||||||
|
"n_dead_tup": "4364",
|
||||||
|
"n_live_tup": "200000",
|
||||||
|
"n_mod_since_analyze": "8901",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "5305",
|
||||||
|
"n_tup_ins": "200000",
|
||||||
|
"n_tup_upd": "8901",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "125261",
|
||||||
|
"idx_tup_fetch": "85299628",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:18.824212-05",
|
||||||
|
"n_dead_tup": "1510",
|
||||||
|
"n_live_tup": "60000",
|
||||||
|
"n_mod_since_analyze": "1594",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "262",
|
||||||
|
"n_tup_ins": "60000",
|
||||||
|
"n_tup_upd": "1594",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"order_line": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "1655",
|
||||||
|
"idx_tup_fetch": "33762",
|
||||||
|
"last_autoanalyze": "2017-11-20 16:00:11.017507-05",
|
||||||
|
"n_dead_tup": "2550",
|
||||||
|
"n_live_tup": "608373",
|
||||||
|
"n_mod_since_analyze": "16230",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "5393",
|
||||||
|
"n_tup_ins": "608373",
|
||||||
|
"n_tup_upd": "7329",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "3",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "627652",
|
||||||
|
"idx_tup_fetch": "627652",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:54.690984-05",
|
||||||
|
"n_dead_tup": "117",
|
||||||
|
"n_live_tup": "60889",
|
||||||
|
"n_mod_since_analyze": "1629",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "662",
|
||||||
|
"n_tup_ins": "60900",
|
||||||
|
"n_tup_upd": "740",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "4",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "1481",
|
||||||
|
"idx_tup_fetch": "1480",
|
||||||
|
"last_autoanalyze": "2017-11-20 16:00:11.217111-05",
|
||||||
|
"n_dead_tup": "751",
|
||||||
|
"n_live_tup": "16964",
|
||||||
|
"n_mod_since_analyze": "1629",
|
||||||
|
"n_tup_del": "740",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "17715",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "2",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "122234",
|
||||||
|
"idx_tup_fetch": "122234",
|
||||||
|
"last_autoanalyze": "2017-11-20 19:23:34.201509-05",
|
||||||
|
"n_dead_tup": "33",
|
||||||
|
"n_live_tup": "20",
|
||||||
|
"n_mod_since_analyze": "0",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "1754",
|
||||||
|
"n_tup_ins": "20",
|
||||||
|
"n_tup_upd": "1754",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "2221",
|
||||||
|
"seq_tup_read": "41522",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"analyze_count": "0",
|
||||||
|
"autoanalyze_count": "1",
|
||||||
|
"autovacuum_count": "0",
|
||||||
|
"idx_scan": "209020",
|
||||||
|
"idx_tup_fetch": "209009",
|
||||||
|
"last_autoanalyze": "2017-11-20 15:59:26.613728-05",
|
||||||
|
"n_dead_tup": "0",
|
||||||
|
"n_live_tup": "102000",
|
||||||
|
"n_mod_since_analyze": "2000",
|
||||||
|
"n_tup_del": "0",
|
||||||
|
"n_tup_hot_upd": "0",
|
||||||
|
"n_tup_ins": "100000",
|
||||||
|
"n_tup_upd": "0",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public",
|
||||||
|
"seq_scan": "1",
|
||||||
|
"seq_tup_read": "0",
|
||||||
|
"vacuum_count": "0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_statio_user_tables": {
|
||||||
|
"history": {
|
||||||
|
"heap_blks_hit": "184380",
|
||||||
|
"heap_blks_read": "746",
|
||||||
|
"relid": "16536",
|
||||||
|
"relname": "history",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"order_line": {
|
||||||
|
"heap_blks_hit": "1869417",
|
||||||
|
"heap_blks_read": "12419",
|
||||||
|
"idx_blks_hit": "1788651",
|
||||||
|
"idx_blks_read": "3708",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"heap_blks_hit": "404486",
|
||||||
|
"heap_blks_read": "80",
|
||||||
|
"idx_blks_hit": "202643",
|
||||||
|
"idx_blks_read": "6",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"heap_blks_hit": "37856",
|
||||||
|
"heap_blks_read": "192",
|
||||||
|
"idx_blks_hit": "38225",
|
||||||
|
"idx_blks_read": "134",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"heap_blks_hit": "1920817",
|
||||||
|
"heap_blks_read": "11757",
|
||||||
|
"idx_blks_hit": "2447522",
|
||||||
|
"idx_blks_read": "1530",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"heap_blks_hit": "1378399",
|
||||||
|
"heap_blks_read": "928",
|
||||||
|
"idx_blks_hit": "3979052",
|
||||||
|
"idx_blks_read": "1881",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"heap_blks_hit": "249754",
|
||||||
|
"heap_blks_read": "3",
|
||||||
|
"idx_blks_hit": "122259",
|
||||||
|
"idx_blks_read": "5",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"heap_blks_hit": "509702",
|
||||||
|
"heap_blks_read": "4542",
|
||||||
|
"idx_blks_hit": "617914",
|
||||||
|
"idx_blks_read": "877",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"heap_blks_hit": "70136669",
|
||||||
|
"heap_blks_read": "13826",
|
||||||
|
"idx_blks_hit": "1411491",
|
||||||
|
"idx_blks_read": "2716",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public",
|
||||||
|
"tidx_blks_hit": "0",
|
||||||
|
"tidx_blks_read": "0",
|
||||||
|
"toast_blks_hit": "0",
|
||||||
|
"toast_blks_read": "0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"database": {
|
||||||
|
"pg_stat_database": {
|
||||||
|
"postgres": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "115229324",
|
||||||
|
"blks_read": "104188",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "12558",
|
||||||
|
"datname": "postgres",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "1",
|
||||||
|
"stats_reset": "2017-11-10 11:14:57.116228-05",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "1818",
|
||||||
|
"tup_fetched": "103355344",
|
||||||
|
"tup_inserted": "2210752",
|
||||||
|
"tup_returned": "110741743",
|
||||||
|
"tup_updated": "32675",
|
||||||
|
"xact_commit": "19082",
|
||||||
|
"xact_rollback": "17"
|
||||||
|
},
|
||||||
|
"tpcc": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "16384",
|
||||||
|
"datname": "tpcc",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
},
|
||||||
|
"template1": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "1",
|
||||||
|
"datname": "template1",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
},
|
||||||
|
"template0": {
|
||||||
|
"blk_read_time": "0",
|
||||||
|
"blk_write_time": "0",
|
||||||
|
"blks_hit": "0",
|
||||||
|
"blks_read": "0",
|
||||||
|
"conflicts": "0",
|
||||||
|
"datid": "12557",
|
||||||
|
"datname": "template0",
|
||||||
|
"deadlocks": "0",
|
||||||
|
"numbackends": "0",
|
||||||
|
"temp_bytes": "0",
|
||||||
|
"temp_files": "0",
|
||||||
|
"tup_deleted": "0",
|
||||||
|
"tup_fetched": "0",
|
||||||
|
"tup_inserted": "0",
|
||||||
|
"tup_returned": "0",
|
||||||
|
"tup_updated": "0",
|
||||||
|
"xact_commit": "0",
|
||||||
|
"xact_rollback": "0"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_stat_database_conflicts": {
|
||||||
|
"postgres": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "12558",
|
||||||
|
"datname": "postgres"
|
||||||
|
},
|
||||||
|
"tpcc": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "16384",
|
||||||
|
"datname": "tpcc"
|
||||||
|
},
|
||||||
|
"template1": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "1",
|
||||||
|
"datname": "template1"
|
||||||
|
},
|
||||||
|
"template0": {
|
||||||
|
"confl_bufferpin": "0",
|
||||||
|
"confl_deadlock": "0",
|
||||||
|
"confl_lock": "0",
|
||||||
|
"confl_snapshot": "0",
|
||||||
|
"confl_tablespace": "0",
|
||||||
|
"datid": "12557",
|
||||||
|
"datname": "template0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"indexes": {
|
||||||
|
"pg_stat_user_indexes": {
|
||||||
|
"order_line": {
|
||||||
|
"idx_scan": "1655",
|
||||||
|
"idx_tup_fetch": "33762",
|
||||||
|
"idx_tup_read": "35698",
|
||||||
|
"indexrelid": "16516",
|
||||||
|
"indexrelname": "order_line_pkey",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"idx_scan": "1481",
|
||||||
|
"idx_tup_fetch": "1480",
|
||||||
|
"idx_tup_read": "2200",
|
||||||
|
"indexrelid": "16521",
|
||||||
|
"indexrelname": "new_order_pkey",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"idx_scan": "644561",
|
||||||
|
"idx_tup_fetch": "644561",
|
||||||
|
"idx_tup_read": "647319",
|
||||||
|
"indexrelid": "16526",
|
||||||
|
"indexrelname": "stock_pkey",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"idx_scan": "616371",
|
||||||
|
"idx_tup_fetch": "616371",
|
||||||
|
"idx_tup_read": "616371",
|
||||||
|
"indexrelid": "16565",
|
||||||
|
"indexrelname": "idx_order",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"idx_scan": "82442",
|
||||||
|
"idx_tup_fetch": "85256809",
|
||||||
|
"idx_tup_read": "85256841",
|
||||||
|
"indexrelid": "16564",
|
||||||
|
"indexrelname": "idx_customer_name",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"idx_scan": "122234",
|
||||||
|
"idx_tup_fetch": "122234",
|
||||||
|
"idx_tup_read": "122234",
|
||||||
|
"indexrelid": "16552",
|
||||||
|
"indexrelname": "district_pkey",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"idx_scan": "209020",
|
||||||
|
"idx_tup_fetch": "209009",
|
||||||
|
"idx_tup_read": "209009",
|
||||||
|
"indexrelid": "16557",
|
||||||
|
"indexrelname": "item_pkey",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"idx_scan": "202634",
|
||||||
|
"idx_tup_fetch": "201331",
|
||||||
|
"idx_tup_read": "202634",
|
||||||
|
"indexrelid": "16562",
|
||||||
|
"indexrelname": "warehouse_pkey",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pg_statio_user_indexes": {
|
||||||
|
"order_line": {
|
||||||
|
"idx_blks_hit": "1788651",
|
||||||
|
"idx_blks_read": "3708",
|
||||||
|
"indexrelid": "16516",
|
||||||
|
"indexrelname": "order_line_pkey",
|
||||||
|
"relid": "16513",
|
||||||
|
"relname": "order_line",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"new_order": {
|
||||||
|
"idx_blks_hit": "38225",
|
||||||
|
"idx_blks_read": "134",
|
||||||
|
"indexrelid": "16521",
|
||||||
|
"indexrelname": "new_order_pkey",
|
||||||
|
"relid": "16518",
|
||||||
|
"relname": "new_order",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"stock": {
|
||||||
|
"idx_blks_hit": "2447522",
|
||||||
|
"idx_blks_read": "1530",
|
||||||
|
"indexrelid": "16526",
|
||||||
|
"indexrelname": "stock_pkey",
|
||||||
|
"relid": "16523",
|
||||||
|
"relname": "stock",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"oorder": {
|
||||||
|
"idx_blks_hit": "3689479",
|
||||||
|
"idx_blks_read": "733",
|
||||||
|
"indexrelid": "16565",
|
||||||
|
"indexrelname": "idx_order",
|
||||||
|
"relid": "16528",
|
||||||
|
"relname": "oorder",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"customer": {
|
||||||
|
"idx_blks_hit": "1151523",
|
||||||
|
"idx_blks_read": "1589",
|
||||||
|
"indexrelid": "16564",
|
||||||
|
"indexrelname": "idx_customer_name",
|
||||||
|
"relid": "16540",
|
||||||
|
"relname": "customer",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"district": {
|
||||||
|
"idx_blks_hit": "122259",
|
||||||
|
"idx_blks_read": "5",
|
||||||
|
"indexrelid": "16552",
|
||||||
|
"indexrelname": "district_pkey",
|
||||||
|
"relid": "16549",
|
||||||
|
"relname": "district",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"item": {
|
||||||
|
"idx_blks_hit": "617914",
|
||||||
|
"idx_blks_read": "877",
|
||||||
|
"indexrelid": "16557",
|
||||||
|
"indexrelname": "item_pkey",
|
||||||
|
"relid": "16554",
|
||||||
|
"relname": "item",
|
||||||
|
"schemaname": "public"
|
||||||
|
},
|
||||||
|
"warehouse": {
|
||||||
|
"idx_blks_hit": "202643",
|
||||||
|
"idx_blks_read": "6",
|
||||||
|
"indexrelid": "16562",
|
||||||
|
"indexrelname": "warehouse_pkey",
|
||||||
|
"relid": "16559",
|
||||||
|
"relname": "warehouse",
|
||||||
|
"schemaname": "public"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"start_time": 1535653369274,
|
||||||
|
"end_time": 1535653559607,
|
||||||
|
"observation_time": 190,
|
||||||
|
"database_type": "postgres",
|
||||||
|
"database_version": "9.3",
|
||||||
|
"workload_name": "workload_name"
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,8 @@
|
||||||
|
{
|
||||||
|
"start_time": 1535653369274,
|
||||||
|
"end_time": 1535653559607,
|
||||||
|
"observation_time": 190,
|
||||||
|
"database_type": "saphana",
|
||||||
|
"database_version": "2.00.023.00.1513691289",
|
||||||
|
"workload_name": "workload_name"
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - ControllerConfiguration.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller;
|
||||||
|
|
||||||
|
/** Controller Configuration. */
|
||||||
|
public class ControllerConfiguration {
|
||||||
|
private DatabaseType dbType;
|
||||||
|
private String dbName;
|
||||||
|
private String dbUsername;
|
||||||
|
private String dbPassword;
|
||||||
|
private String dbURL;
|
||||||
|
private String uploadCode;
|
||||||
|
private String uploadURL;
|
||||||
|
private String workloadName;
|
||||||
|
|
||||||
|
public ControllerConfiguration() {}
|
||||||
|
|
||||||
|
public ControllerConfiguration(
|
||||||
|
String dbName,
|
||||||
|
String dbUsername,
|
||||||
|
String dbPassword,
|
||||||
|
String dbURL,
|
||||||
|
String uploadCode,
|
||||||
|
String uploadURL,
|
||||||
|
String workloadName) {
|
||||||
|
this.dbType = DatabaseType.get(dbName);
|
||||||
|
this.dbName = dbName;
|
||||||
|
this.dbUsername = dbUsername;
|
||||||
|
this.dbPassword = dbPassword;
|
||||||
|
this.dbURL = dbURL;
|
||||||
|
this.uploadCode = uploadCode;
|
||||||
|
this.uploadURL = uploadURL;
|
||||||
|
this.workloadName = workloadName;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Mutators */
|
||||||
|
public void setDBType(DatabaseType dbType) {
|
||||||
|
this.dbType = dbType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDBName(String dbName) {
|
||||||
|
this.dbName = dbName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDBUsername(String dbUsername) {
|
||||||
|
this.dbUsername = dbUsername;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPassword(String dbPassword) {
|
||||||
|
this.dbPassword = dbPassword;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDBURL(String dbURL) {
|
||||||
|
this.dbURL = dbURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUploadCode(String uploadCode) {
|
||||||
|
this.uploadCode = uploadCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUploadURL(String uploadURL) {
|
||||||
|
this.uploadURL = uploadURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWorkloadName(String workloadName) {
|
||||||
|
this.workloadName = workloadName;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Getters */
|
||||||
|
public DatabaseType getDBType() {
|
||||||
|
return this.dbType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDBName() {
|
||||||
|
return this.dbName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDBUsername() {
|
||||||
|
return this.dbUsername;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDBPassword() {
|
||||||
|
return this.dbPassword;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDBURL() {
|
||||||
|
return this.dbURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUploadCode() {
|
||||||
|
return this.uploadCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUploadURL() {
|
||||||
|
return this.uploadURL;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getWorkloadName() {
|
||||||
|
return this.workloadName;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,296 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - Main.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller;
|
||||||
|
|
||||||
|
import com.controller.collectors.DBCollector;
|
||||||
|
import com.controller.collectors.MySQLCollector;
|
||||||
|
import com.controller.collectors.OracleCollector;
|
||||||
|
import com.controller.collectors.PostgresCollector;
|
||||||
|
import com.controller.collectors.SAPHanaCollector;
|
||||||
|
import com.controller.types.JSONSchemaType;
|
||||||
|
import com.controller.util.FileUtil;
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.PrintWriter;
|
||||||
|
import java.io.UnsupportedEncodingException;
|
||||||
|
import java.lang.management.ManagementFactory;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.apache.commons.cli.CommandLine;
|
||||||
|
import org.apache.commons.cli.CommandLineParser;
|
||||||
|
import org.apache.commons.cli.HelpFormatter;
|
||||||
|
import org.apache.commons.cli.Options;
|
||||||
|
import org.apache.commons.cli.ParseException;
|
||||||
|
import org.apache.commons.cli.PosixParser;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
import org.apache.log4j.PropertyConfigurator;
|
||||||
|
import sun.misc.Signal;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Controller main.
|
||||||
|
*
|
||||||
|
* @author Shuli
|
||||||
|
*/
|
||||||
|
public class Main {
|
||||||
|
static final Logger LOG = Logger.getLogger(Main.class);
|
||||||
|
|
||||||
|
// Default output directory name
|
||||||
|
private static final String DEFAULT_DIRECTORY = "output";
|
||||||
|
|
||||||
|
// Default observation period time (5 minutes)
|
||||||
|
private static final int DEFAULT_TIME_SECONDS = -1;
|
||||||
|
|
||||||
|
// Path to JSON schema directory
|
||||||
|
private static final String SCHEMA_PATH = "src/main/java/com/controller/json_validation_schema";
|
||||||
|
|
||||||
|
private static final int TO_MILLISECONDS = 1000;
|
||||||
|
|
||||||
|
private static boolean keepRunning = true;
|
||||||
|
private static boolean firstCollecting = false;
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
|
||||||
|
// Initialize log4j
|
||||||
|
PropertyConfigurator.configure("log4j.properties");
|
||||||
|
|
||||||
|
// Initialize keepRunning
|
||||||
|
keepRunning = true;
|
||||||
|
|
||||||
|
// Initialize firstCollecting
|
||||||
|
firstCollecting = false;
|
||||||
|
|
||||||
|
// Create the command line parser
|
||||||
|
CommandLineParser parser = new PosixParser();
|
||||||
|
Options options = new Options();
|
||||||
|
options.addOption("c", "config", true, "[required] Controller configuration file");
|
||||||
|
options.addOption("t", "time", true, "The observation time in seconds, default is 300s");
|
||||||
|
options.addOption(
|
||||||
|
"d", "directory", true, "Base directory for the result files, default is 'output'");
|
||||||
|
options.addOption("h", "help", true, "Print this help");
|
||||||
|
String configFilePath = null;
|
||||||
|
|
||||||
|
// Parse the command line arguments
|
||||||
|
CommandLine argsLine;
|
||||||
|
try {
|
||||||
|
argsLine = parser.parse(options, args);
|
||||||
|
} catch (ParseException e) {
|
||||||
|
LOG.error("Unable to Parse command line arguments");
|
||||||
|
printUsage(options);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (argsLine.hasOption("h")) {
|
||||||
|
printUsage(options);
|
||||||
|
return;
|
||||||
|
} else if (argsLine.hasOption("c") == false) {
|
||||||
|
LOG.error("Missing configuration file");
|
||||||
|
printUsage(options);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
int time = DEFAULT_TIME_SECONDS;
|
||||||
|
if (argsLine.hasOption("t")) {
|
||||||
|
time = Integer.parseInt(argsLine.getOptionValue("t"));
|
||||||
|
}
|
||||||
|
LOG.info("Experiment time is set to: " + time);
|
||||||
|
|
||||||
|
String outputDirectory = DEFAULT_DIRECTORY;
|
||||||
|
if (argsLine.hasOption("d")) {
|
||||||
|
outputDirectory = argsLine.getOptionValue("d");
|
||||||
|
}
|
||||||
|
LOG.info("Experiment output directory is set to: " + outputDirectory);
|
||||||
|
FileUtil.makeDirIfNotExists(outputDirectory);
|
||||||
|
|
||||||
|
// Parse controller configuration file
|
||||||
|
String configPath = argsLine.getOptionValue("c");
|
||||||
|
File configFile = new File(configPath);
|
||||||
|
|
||||||
|
// Check config format
|
||||||
|
if (!JSONSchemaType.isValidJson(JSONSchemaType.CONFIG, configFile)) {
|
||||||
|
LOG.error("Invalid configuration JSON format");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load configuration file
|
||||||
|
ControllerConfiguration config = null;
|
||||||
|
try {
|
||||||
|
JSONObject input = new JSONObject(FileUtil.readFile(configFile));
|
||||||
|
config =
|
||||||
|
new ControllerConfiguration(
|
||||||
|
input.getString("database_type"),
|
||||||
|
input.getString("username"),
|
||||||
|
input.getString("password"),
|
||||||
|
input.getString("database_url"),
|
||||||
|
input.getString("upload_code"),
|
||||||
|
input.getString("upload_url"),
|
||||||
|
input.getString("workload_name"));
|
||||||
|
} catch (JSONException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
DBCollector collector = getCollector(config);
|
||||||
|
try {
|
||||||
|
// add a signal handler
|
||||||
|
Signal.handle(new Signal("INT"), signal -> firstCollecting = true);
|
||||||
|
File f = new File("pid.txt");
|
||||||
|
|
||||||
|
// get pid of this process and write the pid to a file before recording the start time
|
||||||
|
if (time < 0) {
|
||||||
|
String vmName = ManagementFactory.getRuntimeMXBean().getName();
|
||||||
|
int p = vmName.indexOf("@");
|
||||||
|
int pid = Integer.valueOf(vmName.substring(0, p));
|
||||||
|
try {
|
||||||
|
f.createNewFile();
|
||||||
|
PrintWriter pidWriter = new PrintWriter(f);
|
||||||
|
pidWriter.println(pid);
|
||||||
|
pidWriter.flush();
|
||||||
|
pidWriter.close();
|
||||||
|
} catch (IOException ioe) {
|
||||||
|
ioe.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LOG.info("Output the process pid to pid.txt");
|
||||||
|
|
||||||
|
while (!firstCollecting) {
|
||||||
|
Thread.sleep(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// first collection (before queries)
|
||||||
|
LOG.info("First collection of metrics before experiment");
|
||||||
|
String metricsBefore = collector.collectMetrics();
|
||||||
|
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, metricsBefore)) {
|
||||||
|
LOG.error("Invalid output JSON format (metrics_before)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
PrintWriter metricsWriter =
|
||||||
|
new PrintWriter(FileUtil.joinPath(outputDirectory, "metrics_before.json"), "UTF-8");
|
||||||
|
metricsWriter.println(metricsBefore);
|
||||||
|
metricsWriter.close();
|
||||||
|
|
||||||
|
String knobs = collector.collectParameters();
|
||||||
|
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, knobs)) {
|
||||||
|
LOG.error("Invalid output JSON format (knobs)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
PrintWriter knobsWriter =
|
||||||
|
new PrintWriter(FileUtil.joinPath(outputDirectory, "knobs.json"), "UTF-8");
|
||||||
|
knobsWriter.println(knobs);
|
||||||
|
knobsWriter.close();
|
||||||
|
|
||||||
|
// add a signal handler
|
||||||
|
Signal.handle(new Signal("INT"), signal -> keepRunning = false);
|
||||||
|
|
||||||
|
// record start time
|
||||||
|
long startTime = System.currentTimeMillis();
|
||||||
|
LOG.info("Starting the experiment ...");
|
||||||
|
|
||||||
|
// go to sleep
|
||||||
|
if (time >= 0) {
|
||||||
|
Thread.sleep(time * TO_MILLISECONDS);
|
||||||
|
} else {
|
||||||
|
while (keepRunning) {
|
||||||
|
Thread.sleep(1);
|
||||||
|
}
|
||||||
|
f.delete();
|
||||||
|
}
|
||||||
|
|
||||||
|
long endTime = System.currentTimeMillis();
|
||||||
|
long observationTime = time >= 0 ? time : (endTime - startTime) / TO_MILLISECONDS;
|
||||||
|
LOG.info("Done running the experiment");
|
||||||
|
|
||||||
|
// summary json obj
|
||||||
|
JSONObject summary = null;
|
||||||
|
try {
|
||||||
|
summary = new JSONObject();
|
||||||
|
summary.put("start_time", startTime);
|
||||||
|
summary.put("end_time", endTime);
|
||||||
|
summary.put("observation_time", observationTime);
|
||||||
|
summary.put("database_type", config.getDBName());
|
||||||
|
summary.put("database_version", collector.collectVersion());
|
||||||
|
summary.put("workload_name", config.getWorkloadName());
|
||||||
|
} catch (JSONException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
if (!JSONSchemaType.isValidJson(JSONSchemaType.SUMMARY, summary.toString())) {
|
||||||
|
LOG.error("Invalid summary JSON format");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// write summary JSONObject into a JSON file
|
||||||
|
PrintWriter summaryout =
|
||||||
|
new PrintWriter(FileUtil.joinPath(outputDirectory, "summary.json"), "UTF-8");
|
||||||
|
summaryout.println(JSONUtil.format(summary.toString()));
|
||||||
|
summaryout.close();
|
||||||
|
|
||||||
|
// second collection (after workload execution)
|
||||||
|
LOG.info("Second collection of metrics after experiment");
|
||||||
|
collector = getCollector(config);
|
||||||
|
String metricsAfter = collector.collectMetrics();
|
||||||
|
if (!JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, metricsAfter)) {
|
||||||
|
LOG.error("Invalid output JSON format (metrics_after)");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
PrintWriter metricsWriterFinal =
|
||||||
|
new PrintWriter(FileUtil.joinPath(outputDirectory, "metrics_after.json"), "UTF-8");
|
||||||
|
metricsWriterFinal.println(metricsAfter);
|
||||||
|
metricsWriterFinal.close();
|
||||||
|
} catch (FileNotFoundException | UnsupportedEncodingException | InterruptedException e) {
|
||||||
|
LOG.error("Failed to produce output files");
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
if (config.getUploadURL() != null && !config.getUploadURL().equals("")) {
|
||||||
|
Map<String, String> outfiles = new HashMap<>();
|
||||||
|
outfiles.put("knobs", FileUtil.joinPath(outputDirectory, "knobs.json"));
|
||||||
|
outfiles.put("metrics_before", FileUtil.joinPath(outputDirectory, "metrics_before.json"));
|
||||||
|
outfiles.put("metrics_after", FileUtil.joinPath(outputDirectory, "metrics_after.json"));
|
||||||
|
outfiles.put("summary", FileUtil.joinPath(outputDirectory, "summary.json"));
|
||||||
|
try {
|
||||||
|
ResultUploader.upload(config.getUploadURL(), config.getUploadCode(), outfiles);
|
||||||
|
} catch (IOException ioe) {
|
||||||
|
LOG.warn("Failed to upload results from the controller");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
LOG.warn("Empty upload URL. Skipping upload...");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void printUsage(Options options) {
|
||||||
|
HelpFormatter formatter = new HelpFormatter();
|
||||||
|
formatter.printHelp("controller", options);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static DBCollector getCollector(ControllerConfiguration config) {
|
||||||
|
DBCollector collector = null;
|
||||||
|
switch (config.getDBType()) {
|
||||||
|
case POSTGRES:
|
||||||
|
collector =
|
||||||
|
new PostgresCollector(
|
||||||
|
config.getDBURL(), config.getDBUsername(), config.getDBPassword());
|
||||||
|
break;
|
||||||
|
case MYSQL:
|
||||||
|
collector =
|
||||||
|
new MySQLCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
|
||||||
|
break;
|
||||||
|
case SAPHANA:
|
||||||
|
collector =
|
||||||
|
new SAPHanaCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
|
||||||
|
break;
|
||||||
|
case ORACLE:
|
||||||
|
collector =
|
||||||
|
new OracleCollector(config.getDBURL(), config.getDBUsername(), config.getDBPassword());
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
LOG.error("Invalid database type");
|
||||||
|
throw new RuntimeException("Invalid database type");
|
||||||
|
}
|
||||||
|
return collector;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,62 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - ResultUploader.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.apache.http.HttpEntity;
|
||||||
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||||
|
import org.apache.http.client.methods.HttpPost;
|
||||||
|
import org.apache.http.entity.mime.MultipartEntityBuilder;
|
||||||
|
import org.apache.http.entity.mime.content.FileBody;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.apache.http.util.EntityUtils;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Uploading the result.
|
||||||
|
*
|
||||||
|
* @author Shuli
|
||||||
|
*/
|
||||||
|
public class ResultUploader {
|
||||||
|
public static void upload(String uploadURL, String uploadCode,
|
||||||
|
Map<String, String> files) throws IOException {
|
||||||
|
|
||||||
|
try {
|
||||||
|
List<String> filesToSendNames = new ArrayList<>();
|
||||||
|
List<File> filesToSend = new ArrayList<>();
|
||||||
|
for (String fileName : files.keySet()) {
|
||||||
|
String path = files.get(fileName);
|
||||||
|
filesToSendNames.add(fileName);
|
||||||
|
File f = new File(path);
|
||||||
|
filesToSend.add(f);
|
||||||
|
}
|
||||||
|
CloseableHttpClient httpclient = HttpClients.createDefault();
|
||||||
|
HttpPost httppost = new HttpPost(uploadURL);
|
||||||
|
MultipartEntityBuilder mb =
|
||||||
|
MultipartEntityBuilder.create().addTextBody("upload_code", uploadCode);
|
||||||
|
for (int i = 0; i < filesToSendNames.size(); i++) {
|
||||||
|
mb.addPart(filesToSendNames.get(i), new FileBody(filesToSend.get(i)));
|
||||||
|
}
|
||||||
|
|
||||||
|
HttpEntity reqEntity = mb.build();
|
||||||
|
httppost.setEntity(reqEntity);
|
||||||
|
CloseableHttpResponse response = httpclient.execute(httppost);
|
||||||
|
try {
|
||||||
|
HttpEntity resEntity = response.getEntity();
|
||||||
|
EntityUtils.consume(resEntity);
|
||||||
|
} finally {
|
||||||
|
response.close();
|
||||||
|
}
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new IOException();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - DBCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
public class DBCollector implements DBParameterCollector {
|
||||||
|
|
||||||
|
private static final Logger LOG = Logger.getLogger(DBCollector.class);
|
||||||
|
|
||||||
|
protected static final String JSON_GLOBAL_KEY = "global";
|
||||||
|
protected static final String JSON_LOCAL_KEY = "local";
|
||||||
|
|
||||||
|
protected final Map<String, String> dbParameters = new TreeMap<String, String>();
|
||||||
|
|
||||||
|
protected final Map<String, String> dbMetrics = new TreeMap<String, String>();
|
||||||
|
|
||||||
|
protected final StringBuilder version = new StringBuilder();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasParameters() {
|
||||||
|
return (dbParameters.isEmpty() == false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasMetrics() {
|
||||||
|
return (dbMetrics.isEmpty() == false);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectParameters() {
|
||||||
|
return JSONUtil.format(JSONUtil.toJSONString(dbParameters));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectMetrics() {
|
||||||
|
return JSONUtil.format(JSONUtil.toJSONString(dbMetrics));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectVersion() {
|
||||||
|
return version.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - DBParameterCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
public interface DBParameterCollector {
|
||||||
|
boolean hasParameters();
|
||||||
|
|
||||||
|
boolean hasMetrics();
|
||||||
|
|
||||||
|
String collectParameters();
|
||||||
|
|
||||||
|
String collectMetrics();
|
||||||
|
|
||||||
|
String collectVersion();
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,105 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - MySQLCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
/** */
|
||||||
|
public class MySQLCollector extends DBCollector {
|
||||||
|
private static final Logger LOG = Logger.getLogger(MySQLCollector.class);
|
||||||
|
|
||||||
|
private static final String VERSION_SQL = "SELECT @@GLOBAL.version;";
|
||||||
|
|
||||||
|
private static final String PARAMETERS_SQL = "SHOW VARIABLES;";
|
||||||
|
|
||||||
|
private static final String METRICS_SQL = "SHOW STATUS";
|
||||||
|
|
||||||
|
public MySQLCollector(String oriDBUrl, String username, String password) {
|
||||||
|
try {
|
||||||
|
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
|
||||||
|
Statement s = conn.createStatement();
|
||||||
|
|
||||||
|
// Collect DBMS version
|
||||||
|
ResultSet out = s.executeQuery(VERSION_SQL);
|
||||||
|
if (out.next()) {
|
||||||
|
this.version.append(out.getString(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS parameters
|
||||||
|
out = s.executeQuery(PARAMETERS_SQL);
|
||||||
|
while (out.next()) {
|
||||||
|
dbParameters.put(out.getString(1).toLowerCase(), out.getString(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS internal metrics
|
||||||
|
out = s.executeQuery(METRICS_SQL);
|
||||||
|
while (out.next()) {
|
||||||
|
dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
|
||||||
|
}
|
||||||
|
conn.close();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
LOG.error("Error while collecting DB parameters: " + e.getMessage());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectParameters() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (String k : dbParameters.keySet()) {
|
||||||
|
job.put(k, dbParameters.get(k));
|
||||||
|
}
|
||||||
|
// "global is a fake view_name (a placeholder)"
|
||||||
|
jobLocal.put("global", job);
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectMetrics() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobGlobal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (Map.Entry<String, String> entry : dbMetrics.entrySet()) {
|
||||||
|
job.put(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
// "global" is a a placeholder
|
||||||
|
jobGlobal.put("global", job);
|
||||||
|
stringer.value(jobGlobal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,108 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - OracleCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.Map;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
/** */
|
||||||
|
public class OracleCollector extends DBCollector {
|
||||||
|
private static final Logger LOG = Logger.getLogger(MySQLCollector.class);
|
||||||
|
|
||||||
|
private static final String VERSION_SQL = "select VERSION from product_component_version";
|
||||||
|
|
||||||
|
private static final String PARAMETERS_SQL = "select name, value from v$parameter";
|
||||||
|
|
||||||
|
private static final String PARAMETERS_SQL_WITH_HIDDEN =
|
||||||
|
"select x.ksppinm name, y.ksppstvl value from sys.x$ksppi x, sys.x$ksppcv y where"
|
||||||
|
+ " x.inst_id = userenv('Instance') and y.inst_id = userenv('Instance') and x.indx = y.indx";
|
||||||
|
|
||||||
|
private static final String METRICS_SQL = "select name, value from v$sysstat";
|
||||||
|
|
||||||
|
public OracleCollector(String oriDBUrl, String username, String password) {
|
||||||
|
try {
|
||||||
|
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
|
||||||
|
Statement statement = conn.createStatement();
|
||||||
|
// Collect DBMS version
|
||||||
|
ResultSet out = statement.executeQuery(VERSION_SQL);
|
||||||
|
if (out.next()) {
|
||||||
|
this.version.append(out.getString(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS parameters
|
||||||
|
out = statement.executeQuery(PARAMETERS_SQL_WITH_HIDDEN);
|
||||||
|
while (out.next()) {
|
||||||
|
dbParameters.put(out.getString(1).toLowerCase(), out.getString(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS internal metrics
|
||||||
|
out = statement.executeQuery(METRICS_SQL);
|
||||||
|
while (out.next()) {
|
||||||
|
dbMetrics.put(out.getString(1).toLowerCase(), out.getString(2));
|
||||||
|
}
|
||||||
|
conn.close();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
LOG.error("Error while collecting DB parameters: " + e.getMessage());
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectParameters() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (Map.Entry<String, String> entry : dbParameters.entrySet()) {
|
||||||
|
job.put(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
// "global is a fake view_name (a placeholder)"
|
||||||
|
jobLocal.put("global", job);
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectMetrics() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobGlobal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (Map.Entry<String, String> entry : dbMetrics.entrySet()) {
|
||||||
|
job.put(entry.getKey(), entry.getValue());
|
||||||
|
}
|
||||||
|
// "global" is a placeholder
|
||||||
|
jobGlobal.put("global", job);
|
||||||
|
stringer.value(jobGlobal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,237 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - PostgresCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.ResultSetMetaData;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
public class PostgresCollector extends DBCollector {
|
||||||
|
private static final Logger LOG = Logger.getLogger(PostgresCollector.class);
|
||||||
|
|
||||||
|
private static final String VERSION_SQL = "SELECT version();";
|
||||||
|
|
||||||
|
private static final String PARAMETERS_SQL = "SHOW ALL;";
|
||||||
|
|
||||||
|
private boolean oldVersion = false;
|
||||||
|
|
||||||
|
private static final String[] PG_STAT_VIEWS = {
|
||||||
|
"pg_stat_archiver", "pg_stat_bgwriter", "pg_stat_database",
|
||||||
|
"pg_stat_database_conflicts", "pg_stat_user_tables", "pg_statio_user_tables",
|
||||||
|
"pg_stat_user_indexes", "pg_statio_user_indexes"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static final String[] PG_STAT_VIEWS_OLD_VERSION = {
|
||||||
|
"pg_stat_bgwriter", "pg_stat_database",
|
||||||
|
"pg_stat_database_conflicts", "pg_stat_user_tables", "pg_statio_user_tables",
|
||||||
|
"pg_stat_user_indexes", "pg_statio_user_indexes"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static final String[] PG_STAT_VIEWS_LOCAL_DATABASE = {
|
||||||
|
"pg_stat_database", "pg_stat_database_conflicts"
|
||||||
|
};
|
||||||
|
private static final String PG_STAT_VIEWS_LOCAL_DATABASE_KEY = "datname";
|
||||||
|
private static final String[] PG_STAT_VIEWS_LOCAL_TABLE = {
|
||||||
|
"pg_stat_user_tables", "pg_statio_user_tables"
|
||||||
|
};
|
||||||
|
private static final String PG_STAT_VIEWS_LOCAL_TABLE_KEY = "relname";
|
||||||
|
private static final String[] PG_STAT_VIEWS_LOCAL_INDEXES = {
|
||||||
|
"pg_stat_user_indexes", "pg_statio_user_indexes"
|
||||||
|
};
|
||||||
|
private static final String PG_STAT_VIEWS_LOCAL_INDEXES_KEY = "relname";
|
||||||
|
|
||||||
|
private final Map<String, List<Map<String, String>>> pgMetrics;
|
||||||
|
|
||||||
|
public PostgresCollector(String oriDBUrl, String username, String password) {
|
||||||
|
pgMetrics = new HashMap<>();
|
||||||
|
try {
|
||||||
|
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
|
||||||
|
|
||||||
|
Statement s = conn.createStatement();
|
||||||
|
|
||||||
|
// Collect DBMS version
|
||||||
|
ResultSet out = s.executeQuery(VERSION_SQL);
|
||||||
|
if (out.next()) {
|
||||||
|
String[] outStr = out.getString(1).split(" ");
|
||||||
|
String[] verStr = outStr[1].split("\\.");
|
||||||
|
this.version.append(verStr[0]);
|
||||||
|
this.version.append(".");
|
||||||
|
this.version.append(verStr[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS parameters
|
||||||
|
out = s.executeQuery(PARAMETERS_SQL);
|
||||||
|
while (out.next()) {
|
||||||
|
dbParameters.put(out.getString("name"), out.getString("setting"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS internal metrics
|
||||||
|
String[] pgStatViews = PG_STAT_VIEWS;
|
||||||
|
if (Float.parseFloat(this.version.toString()) < 9.4) {
|
||||||
|
this.oldVersion = true;
|
||||||
|
pgStatViews = PG_STAT_VIEWS_OLD_VERSION;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (String viewName : pgStatViews) {
|
||||||
|
out = s.executeQuery("SELECT * FROM " + viewName);
|
||||||
|
pgMetrics.put(viewName, getMetrics(out));
|
||||||
|
}
|
||||||
|
conn.close();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
LOG.error("Error while collecting DB parameters: " + e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasMetrics() {
|
||||||
|
return (pgMetrics.isEmpty() == false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject genMapJSONObj(Map<String, String> mapin) {
|
||||||
|
JSONObject res = new JSONObject();
|
||||||
|
try {
|
||||||
|
for (String key : mapin.keySet()) {
|
||||||
|
res.put(key, mapin.get(key));
|
||||||
|
}
|
||||||
|
} catch (JSONException je) {
|
||||||
|
LOG.error(je);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject genLocalJSONObj(String viewName, String jsonKeyName) {
|
||||||
|
JSONObject thisViewObj = new JSONObject();
|
||||||
|
List<Map<String, String>> thisViewList = pgMetrics.get(viewName);
|
||||||
|
try {
|
||||||
|
for (Map<String, String> dbmap : thisViewList) {
|
||||||
|
String jsonkey = dbmap.get(jsonKeyName);
|
||||||
|
thisViewObj.put(jsonkey, genMapJSONObj(dbmap));
|
||||||
|
}
|
||||||
|
} catch (JSONException je) {
|
||||||
|
LOG.error(je);
|
||||||
|
}
|
||||||
|
return thisViewObj;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectMetrics() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
// create global objects for two views: "pg_stat_archiver" and "pg_stat_bgwriter"
|
||||||
|
JSONObject jobGlobal = new JSONObject();
|
||||||
|
// "pg_stat_archiver" (only one instance in the list) >= version 9.4
|
||||||
|
if (!this.oldVersion) {
|
||||||
|
Map<String, String> archiverList = pgMetrics.get("pg_stat_archiver").get(0);
|
||||||
|
jobGlobal.put("pg_stat_archiver", genMapJSONObj(archiverList));
|
||||||
|
}
|
||||||
|
|
||||||
|
// "pg_stat_bgwriter" (only one instance in the list)
|
||||||
|
Map<String, String> bgwriterList = pgMetrics.get("pg_stat_bgwriter").get(0);
|
||||||
|
jobGlobal.put("pg_stat_bgwriter", genMapJSONObj(bgwriterList));
|
||||||
|
|
||||||
|
// add global json object
|
||||||
|
stringer.value(jobGlobal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
// create local objects for the rest of the views
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
|
||||||
|
// "table"
|
||||||
|
JSONObject jobTable = new JSONObject();
|
||||||
|
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_TABLE.length; i++) {
|
||||||
|
String viewName = PG_STAT_VIEWS_LOCAL_TABLE[i];
|
||||||
|
String jsonKeyName = PG_STAT_VIEWS_LOCAL_TABLE_KEY;
|
||||||
|
jobTable.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
jobLocal.put("table", jobTable);
|
||||||
|
|
||||||
|
// "database"
|
||||||
|
JSONObject jobDatabase = new JSONObject();
|
||||||
|
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_DATABASE.length; i++) {
|
||||||
|
String viewName = PG_STAT_VIEWS_LOCAL_DATABASE[i];
|
||||||
|
String jsonKeyName = PG_STAT_VIEWS_LOCAL_DATABASE_KEY;
|
||||||
|
jobDatabase.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
jobLocal.put("database", jobDatabase);
|
||||||
|
|
||||||
|
// "indexes"
|
||||||
|
JSONObject jobIndexes = new JSONObject();
|
||||||
|
for (int i = 0; i < PG_STAT_VIEWS_LOCAL_INDEXES.length; i++) {
|
||||||
|
String viewName = PG_STAT_VIEWS_LOCAL_INDEXES[i];
|
||||||
|
String jsonKeyName = PG_STAT_VIEWS_LOCAL_INDEXES_KEY;
|
||||||
|
jobIndexes.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
jobLocal.put("indexes", jobIndexes);
|
||||||
|
|
||||||
|
// add local json object
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.endObject();
|
||||||
|
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Map<String, String>> getMetrics(ResultSet out) throws SQLException {
|
||||||
|
ResultSetMetaData metadata = out.getMetaData();
|
||||||
|
int numColumns = metadata.getColumnCount();
|
||||||
|
String[] columnNames = new String[numColumns];
|
||||||
|
for (int i = 0; i < numColumns; ++i) {
|
||||||
|
columnNames[i] = metadata.getColumnName(i + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Map<String, String>> metrics = new ArrayList<Map<String, String>>();
|
||||||
|
while (out.next()) {
|
||||||
|
Map<String, String> metricMap = new TreeMap<String, String>();
|
||||||
|
for (int i = 0; i < numColumns; ++i) {
|
||||||
|
metricMap.put(columnNames[i], out.getString(i + 1));
|
||||||
|
}
|
||||||
|
metrics.add(metricMap);
|
||||||
|
}
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectParameters() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (String k : dbParameters.keySet()) {
|
||||||
|
job.put(k, dbParameters.get(k));
|
||||||
|
}
|
||||||
|
// "global is a fake view_name (a placeholder)"
|
||||||
|
jobLocal.put("global", job);
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,231 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - SAPHanaCollector.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.util.JSONUtil;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.sql.Connection;
|
||||||
|
import java.sql.DriverManager;
|
||||||
|
import java.sql.ResultSet;
|
||||||
|
import java.sql.ResultSetMetaData;
|
||||||
|
import java.sql.SQLException;
|
||||||
|
import java.sql.Statement;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.TreeMap;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
public class SAPHanaCollector extends DBCollector {
|
||||||
|
private static final Logger LOG = Logger.getLogger(SAPHanaCollector.class);
|
||||||
|
|
||||||
|
private static final String VERSION_SQL = "SELECT VERSION from M_DATABASE";
|
||||||
|
|
||||||
|
private static final String PARAMETERS_SQL = "Select * from M_INIFILE_CONTENTS";
|
||||||
|
|
||||||
|
private static final String[] SAP_SYS_VIEW = {
|
||||||
|
"m_host_agent_metrics",
|
||||||
|
"m_caches",
|
||||||
|
"m_disk_usage",
|
||||||
|
"m_garbage_collection_statistics",
|
||||||
|
"m_host_resource_utilization",
|
||||||
|
"m_data_volumes"
|
||||||
|
};
|
||||||
|
|
||||||
|
private static final String[] SAP_SYS_LOCAL_VIEW = {"m_table_statistics", "m_rs_indexes"};
|
||||||
|
private static final String[] SAP_SYS_VIEW_GLOBAL = {
|
||||||
|
"m_host_agent_metrics",
|
||||||
|
"m_caches",
|
||||||
|
"m_disk_usage",
|
||||||
|
"m_garbage_collection_statistics",
|
||||||
|
"m_host_resource_utilization",
|
||||||
|
"m_data_volumes"
|
||||||
|
};
|
||||||
|
private static final String[] SAP_SYS_VIEW_GLOBAL_KEY = {
|
||||||
|
"instance_id", "cache_id", "usage_type", "store_type", "host", "volume_id"
|
||||||
|
};
|
||||||
|
private static final String[] SAP_SYS_VIEW_LOCAL_TABLE = {"m_table_statistics"};
|
||||||
|
private static final String[] SAP_SYS_VIEW_LOCAL_TABLE_KEY = {"table_name"};
|
||||||
|
private static final String[] SAP_SYS_VIEW_LOCAL_INDEXES = {"m_rs_indexes"};
|
||||||
|
private static final String[] SAP_SYS_VIEW_LOCAL_INDEXES_KEY = {"index_name"};
|
||||||
|
|
||||||
|
private final Map<String, List<Map<String, String>>> pgMetrics;
|
||||||
|
|
||||||
|
public SAPHanaCollector(String oriDBUrl, String username, String password) {
|
||||||
|
pgMetrics = new HashMap<>();
|
||||||
|
try {
|
||||||
|
Connection conn = DriverManager.getConnection(oriDBUrl, username, password);
|
||||||
|
Statement s = conn.createStatement();
|
||||||
|
|
||||||
|
// Collect DBMS version
|
||||||
|
ResultSet out = s.executeQuery(VERSION_SQL);
|
||||||
|
if (out.next()) {
|
||||||
|
this.version.append(out.getString(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS parameters
|
||||||
|
out = s.executeQuery(PARAMETERS_SQL);
|
||||||
|
while (out.next()) {
|
||||||
|
dbParameters.put(
|
||||||
|
"("
|
||||||
|
+ out.getString("FILE_NAME")
|
||||||
|
+ ","
|
||||||
|
+ out.getString("LAYER_NAME")
|
||||||
|
+ ","
|
||||||
|
+ out.getString("TENANT_NAME")
|
||||||
|
+ ","
|
||||||
|
+ out.getString("HOST")
|
||||||
|
+ ","
|
||||||
|
+ out.getString("SECTION")
|
||||||
|
+ ","
|
||||||
|
+ out.getString("KEY")
|
||||||
|
+ ")",
|
||||||
|
out.getString("VALUE"));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Collect DBMS internal metrics
|
||||||
|
for (String viewName : SAP_SYS_VIEW) {
|
||||||
|
out = s.executeQuery("SELECT * FROM " + viewName);
|
||||||
|
pgMetrics.put(viewName, getMetrics(out));
|
||||||
|
}
|
||||||
|
for (String viewName : SAP_SYS_LOCAL_VIEW) {
|
||||||
|
out = s.executeQuery("SELECT * FROM " + viewName + " where schema_name = 'SYSTEM' ");
|
||||||
|
pgMetrics.put(viewName, getMetrics(out));
|
||||||
|
}
|
||||||
|
conn.close();
|
||||||
|
} catch (SQLException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
LOG.error("Error while collecting DB parameters: " + e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasMetrics() {
|
||||||
|
return (pgMetrics.isEmpty() == false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject genMapJSONObj(Map<String, String> mapin) {
|
||||||
|
JSONObject res = new JSONObject();
|
||||||
|
try {
|
||||||
|
for (String key : mapin.keySet()) {
|
||||||
|
res.put(key, mapin.get(key));
|
||||||
|
}
|
||||||
|
} catch (JSONException je) {
|
||||||
|
LOG.error(je);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
private JSONObject genLocalJSONObj(String viewName, String jsonKeyName) {
|
||||||
|
JSONObject thisViewObj = new JSONObject();
|
||||||
|
List<Map<String, String>> thisViewList = pgMetrics.get(viewName);
|
||||||
|
try {
|
||||||
|
for (Map<String, String> dbmap : thisViewList) {
|
||||||
|
String jsonkey = dbmap.get(jsonKeyName);
|
||||||
|
thisViewObj.put(jsonkey, genMapJSONObj(dbmap));
|
||||||
|
}
|
||||||
|
} catch (JSONException je) {
|
||||||
|
LOG.error(je);
|
||||||
|
}
|
||||||
|
return thisViewObj;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectMetrics() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobGlobal = new JSONObject();
|
||||||
|
|
||||||
|
JSONObject jobMetric = new JSONObject();
|
||||||
|
for (int i = 0; i < SAP_SYS_VIEW_GLOBAL.length; i++) {
|
||||||
|
String viewName = SAP_SYS_VIEW_GLOBAL[i];
|
||||||
|
String jsonKeyName = SAP_SYS_VIEW_GLOBAL_KEY[i];
|
||||||
|
jobGlobal.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
// add global json object
|
||||||
|
stringer.value(jobGlobal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
|
||||||
|
// create local objects for the rest of the views
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
|
||||||
|
// "table"
|
||||||
|
JSONObject jobTable = new JSONObject();
|
||||||
|
for (int i = 0; i < SAP_SYS_VIEW_LOCAL_TABLE.length; i++) {
|
||||||
|
String viewName = SAP_SYS_VIEW_LOCAL_TABLE[i];
|
||||||
|
String jsonKeyName = SAP_SYS_VIEW_LOCAL_TABLE_KEY[i];
|
||||||
|
jobTable.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
jobLocal.put("table", jobTable);
|
||||||
|
|
||||||
|
// "indexes"
|
||||||
|
JSONObject jobIndexes = new JSONObject();
|
||||||
|
for (int i = 0; i < SAP_SYS_VIEW_LOCAL_INDEXES.length; i++) {
|
||||||
|
String viewName = SAP_SYS_VIEW_LOCAL_INDEXES[i];
|
||||||
|
String jsonKeyName = SAP_SYS_VIEW_LOCAL_INDEXES_KEY[i];
|
||||||
|
jobIndexes.put(viewName, genLocalJSONObj(viewName, jsonKeyName));
|
||||||
|
}
|
||||||
|
jobLocal.put("indexes", jobIndexes);
|
||||||
|
|
||||||
|
// add local json object
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.endObject();
|
||||||
|
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<Map<String, String>> getMetrics(ResultSet out) throws SQLException {
|
||||||
|
ResultSetMetaData metadata = out.getMetaData();
|
||||||
|
int numColumns = metadata.getColumnCount();
|
||||||
|
String[] columnNames = new String[numColumns];
|
||||||
|
for (int i = 0; i < numColumns; ++i) {
|
||||||
|
columnNames[i] = metadata.getColumnName(i + 1).toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Map<String, String>> metrics = new ArrayList<Map<String, String>>();
|
||||||
|
while (out.next()) {
|
||||||
|
Map<String, String> metricMap = new TreeMap<String, String>();
|
||||||
|
for (int i = 0; i < numColumns; ++i) {
|
||||||
|
metricMap.put(columnNames[i], out.getString(i + 1));
|
||||||
|
}
|
||||||
|
metrics.add(metricMap);
|
||||||
|
}
|
||||||
|
return metrics;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String collectParameters() {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
stringer.object();
|
||||||
|
stringer.key(JSON_GLOBAL_KEY);
|
||||||
|
JSONObject jobLocal = new JSONObject();
|
||||||
|
JSONObject job = new JSONObject();
|
||||||
|
for (String k : dbParameters.keySet()) {
|
||||||
|
job.put(k, dbParameters.get(k));
|
||||||
|
}
|
||||||
|
// "global is a fake view_name (a placeholder)"
|
||||||
|
jobLocal.put("global", job);
|
||||||
|
stringer.value(jobLocal);
|
||||||
|
stringer.key(JSON_LOCAL_KEY);
|
||||||
|
stringer.value(null);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException jsonexn) {
|
||||||
|
jsonexn.printStackTrace();
|
||||||
|
}
|
||||||
|
return JSONUtil.format(stringer.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,38 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"title": "summary of collector output",
|
||||||
|
"description": "config files: user input",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"database_type": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"database_url": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"username" :{
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"password": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"upload_code": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"upload_url": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"workload_name": {
|
||||||
|
"type" : "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"database_type",
|
||||||
|
"database_url",
|
||||||
|
"username",
|
||||||
|
"password",
|
||||||
|
"upload_code",
|
||||||
|
"upload_url",
|
||||||
|
"workload_name"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,47 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"title": "collector output",
|
||||||
|
"description": "Collected metrics or knobs info",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"global" : {
|
||||||
|
"type": ["object", "null"],
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "object",
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
"local" : {
|
||||||
|
"type": ["object", "null"],
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "object",
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "object",
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "object",
|
||||||
|
"patternProperties" : {
|
||||||
|
"^[A-Za-z0-9 -_]" : {
|
||||||
|
"type" : "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": ["global", "local"]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
{
|
||||||
|
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||||
|
"title": "summary of collector output",
|
||||||
|
"description": "output summary",
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"database_type": {
|
||||||
|
"type" : "string"
|
||||||
|
},
|
||||||
|
"start_time": {
|
||||||
|
"type" : "number"
|
||||||
|
},
|
||||||
|
"observation_time" :{
|
||||||
|
"type" : "number"
|
||||||
|
},
|
||||||
|
"end_time": {
|
||||||
|
"type" : "number"
|
||||||
|
},
|
||||||
|
"database_version": {
|
||||||
|
"type" : "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"database_type",
|
||||||
|
"start_time",
|
||||||
|
"observation_time",
|
||||||
|
"end_time",
|
||||||
|
"database_version"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,66 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - DatabaseType.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller;
|
||||||
|
|
||||||
|
import java.util.EnumSet;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/** Database Type. */
|
||||||
|
public enum DatabaseType {
|
||||||
|
|
||||||
|
/** Parameters: (1) JDBC Driver String */
|
||||||
|
MYSQL("com.mysql.jdbc.Driver"),
|
||||||
|
MYROCKS("com.mysql.jdbc.Driver"),
|
||||||
|
POSTGRES("org.postgresql.Driver"),
|
||||||
|
SAPHANA("com.sap.db.jdbc.Driver"),
|
||||||
|
ORACLE("oracle.jdbc.driver.OracleDriver");
|
||||||
|
|
||||||
|
private DatabaseType(String driver) {
|
||||||
|
this.driver = driver;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This is the suggested driver string to use in the configuration xml This corresponds to the
|
||||||
|
* <B>'driver'</b> attribute.
|
||||||
|
*/
|
||||||
|
private final String driver;
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------
|
||||||
|
// ACCESSORS
|
||||||
|
// ----------------------------------------------------------------
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the suggested driver string to use for the given database type
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public String getSuggestedDriver() {
|
||||||
|
return (this.driver);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----------------------------------------------------------------
|
||||||
|
// STATIC METHODS + MEMBERS
|
||||||
|
// ----------------------------------------------------------------
|
||||||
|
|
||||||
|
protected static final Map<Integer, DatabaseType> idx_lookup =
|
||||||
|
new HashMap<Integer, DatabaseType>();
|
||||||
|
protected static final Map<String, DatabaseType> name_lookup =
|
||||||
|
new HashMap<String, DatabaseType>();
|
||||||
|
|
||||||
|
static {
|
||||||
|
for (DatabaseType vt : EnumSet.allOf(DatabaseType.class)) {
|
||||||
|
DatabaseType.idx_lookup.put(vt.ordinal(), vt);
|
||||||
|
DatabaseType.name_lookup.put(vt.name().toUpperCase(), vt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DatabaseType get(String name) {
|
||||||
|
DatabaseType ret = DatabaseType.name_lookup.get(name.toUpperCase());
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - JSONSchemaType.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.types;
|
||||||
|
|
||||||
|
import com.controller.util.FileUtil;
|
||||||
|
import com.controller.util.ValidationUtils;
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
|
||||||
|
import com.github.fge.jsonschema.main.JsonSchema;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
public enum JSONSchemaType {
|
||||||
|
|
||||||
|
/** Parameters: (1) schema filename */
|
||||||
|
OUTPUT("schema.json"),
|
||||||
|
CONFIG("config_schema.json"),
|
||||||
|
SUMMARY("summary_schema.json");
|
||||||
|
|
||||||
|
// Path to JSON schema directory
|
||||||
|
private static final String SCHEMA_PATH = "src/main/java/com/controller/json_validation_schema";
|
||||||
|
|
||||||
|
private final JsonSchema schema;
|
||||||
|
|
||||||
|
private JSONSchemaType(String fileName) {
|
||||||
|
JsonSchema newSchema = null;
|
||||||
|
String configPath = FileUtil.joinPath(SCHEMA_PATH, fileName);
|
||||||
|
try {
|
||||||
|
newSchema = ValidationUtils.getSchemaNode(new File(configPath));
|
||||||
|
} catch (IOException | ProcessingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
this.schema = newSchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public JsonSchema getSchema() {
|
||||||
|
return this.schema;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isValidJson(JSONSchemaType schemaType, String jsonString) {
|
||||||
|
try {
|
||||||
|
JsonNode jsonNode = ValidationUtils.getJsonNode(jsonString);
|
||||||
|
return ValidationUtils.isJsonValid(schemaType.getSchema(), jsonNode);
|
||||||
|
} catch (IOException | ProcessingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean isValidJson(JSONSchemaType schemaType, File jsonFile) {
|
||||||
|
try {
|
||||||
|
JsonNode jsonNode = ValidationUtils.getJsonNode(jsonFile);
|
||||||
|
return ValidationUtils.isJsonValid(schemaType.getSchema(), jsonNode);
|
||||||
|
} catch (IOException | ProcessingException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,297 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - ClassUtil.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import java.lang.reflect.Constructor;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.ParameterizedType;
|
||||||
|
import java.lang.reflect.Type;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Set;
|
||||||
|
import org.apache.commons.collections15.CollectionUtils;
|
||||||
|
import org.apache.commons.lang.ClassUtils;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
/** @author pavlo */
|
||||||
|
public abstract class ClassUtil {
|
||||||
|
private static final Logger LOG = Logger.getLogger(ClassUtil.class);
|
||||||
|
|
||||||
|
private static final Class<?>[] EMPTY_ARRAY = new Class[] {};
|
||||||
|
|
||||||
|
private static final Map<Class<?>, List<Class<?>>> CACHE_getSuperClasses =
|
||||||
|
new HashMap<Class<?>, List<Class<?>>>();
|
||||||
|
private static final Map<Class<?>, Set<Class<?>>> CACHE_getInterfaceClasses =
|
||||||
|
new HashMap<Class<?>, Set<Class<?>>>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the given object is an array (primitve or native).
|
||||||
|
* http://www.java2s.com/Code/Java/Reflection/Checkifthegivenobjectisanarrayprimitveornative.htm
|
||||||
|
*
|
||||||
|
* @param obj Object to test.
|
||||||
|
* @return True of the object is an array.
|
||||||
|
*/
|
||||||
|
public static boolean isArray(final Object obj) {
|
||||||
|
return (obj != null ? obj.getClass().isArray() : false);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean[] isArray(final Object[] objs) {
|
||||||
|
boolean[] isArray = new boolean[objs.length];
|
||||||
|
for (int i = 0; i < objs.length; i++) {
|
||||||
|
isArray[i] = ClassUtil.isArray(objs[i]);
|
||||||
|
} // FOR
|
||||||
|
return (isArray);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Convert a Enum array to a Field array This assumes that the name of each Enum element
|
||||||
|
* corresponds to a data member in the clas
|
||||||
|
*
|
||||||
|
* @param <E>
|
||||||
|
* @param clazz
|
||||||
|
* @param members
|
||||||
|
* @return
|
||||||
|
* @throws NoSuchFieldException
|
||||||
|
*/
|
||||||
|
public static <E extends Enum<?>> Field[] getFieldsFromMembersEnum(Class<?> clazz, E[] members)
|
||||||
|
throws NoSuchFieldException {
|
||||||
|
Field[] fields = new Field[members.length];
|
||||||
|
for (int i = 0; i < members.length; i++) {
|
||||||
|
fields[i] = clazz.getDeclaredField(members[i].name().toLowerCase());
|
||||||
|
} // FOR
|
||||||
|
return (fields);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the generic types for the given field
|
||||||
|
*
|
||||||
|
* @param field
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static List<Class<?>> getGenericTypes(Field field) {
|
||||||
|
ArrayList<Class<?>> genericClasses = new ArrayList<Class<?>>();
|
||||||
|
Type gtype = field.getGenericType();
|
||||||
|
if (gtype instanceof ParameterizedType) {
|
||||||
|
ParameterizedType ptype = (ParameterizedType) gtype;
|
||||||
|
getGenericTypesImpl(ptype, genericClasses);
|
||||||
|
}
|
||||||
|
return (genericClasses);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void getGenericTypesImpl(ParameterizedType ptype, List<Class<?>> classes) {
|
||||||
|
// list the actual type arguments
|
||||||
|
for (Type t : ptype.getActualTypeArguments()) {
|
||||||
|
if (t instanceof Class) {
|
||||||
|
// System.err.println("C: " + t);
|
||||||
|
classes.add((Class<?>) t);
|
||||||
|
} else if (t instanceof ParameterizedType) {
|
||||||
|
ParameterizedType next = (ParameterizedType) t;
|
||||||
|
// System.err.println("PT: " + next);
|
||||||
|
classes.add((Class<?>) next.getRawType());
|
||||||
|
getGenericTypesImpl(next, classes);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return an ordered list of all the sub-classes for a given class Useful when dealing with
|
||||||
|
* generics
|
||||||
|
*
|
||||||
|
* @param elementClass
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static List<Class<?>> getSuperClasses(Class<?> elementClass) {
|
||||||
|
List<Class<?>> ret = ClassUtil.CACHE_getSuperClasses.get(elementClass);
|
||||||
|
if (ret == null) {
|
||||||
|
ret = new ArrayList<Class<?>>();
|
||||||
|
while (elementClass != null) {
|
||||||
|
ret.add(elementClass);
|
||||||
|
elementClass = elementClass.getSuperclass();
|
||||||
|
} // WHILE
|
||||||
|
ret = Collections.unmodifiableList(ret);
|
||||||
|
ClassUtil.CACHE_getSuperClasses.put(elementClass, ret);
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get a set of all of the interfaces that the element_class implements
|
||||||
|
*
|
||||||
|
* @param elementClass
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public static Collection<Class<?>> getInterfaces(Class<?> elementClass) {
|
||||||
|
Set<Class<?>> ret = ClassUtil.CACHE_getInterfaceClasses.get(elementClass);
|
||||||
|
if (ret == null) {
|
||||||
|
// ret = new HashSet<Class<?>>();
|
||||||
|
// Queue<Class<?>> queue = new LinkedList<Class<?>>();
|
||||||
|
// queue.add(element_class);
|
||||||
|
// while (!queue.isEmpty()) {
|
||||||
|
// Class<?> current = queue.poll();
|
||||||
|
// for (Class<?> i : current.getInterfaces()) {
|
||||||
|
// ret.add(i);
|
||||||
|
// queue.add(i);
|
||||||
|
// } // FOR
|
||||||
|
// } // WHILE
|
||||||
|
ret = new HashSet<Class<?>>(ClassUtils.getAllInterfaces(elementClass));
|
||||||
|
if (elementClass.isInterface()) {
|
||||||
|
ret.add(elementClass);
|
||||||
|
}
|
||||||
|
ret = Collections.unmodifiableSet(ret);
|
||||||
|
ClassUtil.CACHE_getInterfaceClasses.put(elementClass, ret);
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public static <T> T newInstance(String className, Object[] params, Class<?>[] classes) {
|
||||||
|
return ((T) ClassUtil.newInstance(ClassUtil.getClass(className), params, classes));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T newInstance(Class<T> targetClass, Object[] params, Class<?>[] classes) {
|
||||||
|
// Class<?> const_params[] = new Class<?>[params.length];
|
||||||
|
// for (int i = 0; i < params.length; i++) {
|
||||||
|
// const_params[i] = params[i].getClass();
|
||||||
|
// System.err.println("[" + i + "] " + params[i] + " " + params[i].getClass());
|
||||||
|
// } // FOR
|
||||||
|
|
||||||
|
Constructor<T> constructor = ClassUtil.getConstructor(targetClass, classes);
|
||||||
|
T ret = null;
|
||||||
|
try {
|
||||||
|
ret = constructor.newInstance(params);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Failed to create new instance of " + targetClass.getSimpleName(), ex);
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create an object for the given class and initialize it from conf
|
||||||
|
*
|
||||||
|
* @param theClass class of which an object is created
|
||||||
|
* @param expected the expected parent class or interface
|
||||||
|
* @return a new object
|
||||||
|
*/
|
||||||
|
public static <T> T newInstance(Class<?> theClass, Class<T> expected) {
|
||||||
|
T result;
|
||||||
|
try {
|
||||||
|
if (!expected.isAssignableFrom(theClass)) {
|
||||||
|
throw new Exception(
|
||||||
|
"Specified class "
|
||||||
|
+ theClass.getName()
|
||||||
|
+ ""
|
||||||
|
+ "does not extend/implement "
|
||||||
|
+ expected.getName());
|
||||||
|
}
|
||||||
|
Class<? extends T> clazz = (Class<? extends T>) theClass;
|
||||||
|
Constructor<? extends T> meth = clazz.getDeclaredConstructor(EMPTY_ARRAY);
|
||||||
|
meth.setAccessible(true);
|
||||||
|
result = meth.newInstance();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T newInstance(String className, Class<T> expected)
|
||||||
|
throws ClassNotFoundException {
|
||||||
|
return newInstance(getClass(className), expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param <T>
|
||||||
|
* @param targetClass
|
||||||
|
* @param params
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public static <T> Constructor<T> getConstructor(Class<T> targetClass, Class<?>... params) {
|
||||||
|
NoSuchMethodException error = null;
|
||||||
|
try {
|
||||||
|
return (targetClass.getConstructor(params));
|
||||||
|
} catch (NoSuchMethodException ex) {
|
||||||
|
// The first time we get this it can be ignored
|
||||||
|
// We'll try to be nice and find a match for them
|
||||||
|
error = ex;
|
||||||
|
}
|
||||||
|
assert (error != null);
|
||||||
|
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("TARGET CLASS: " + targetClass);
|
||||||
|
LOG.debug("TARGET PARAMS: " + Arrays.toString(params));
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Class<?>>[] paramSuper = (List<Class<?>>[]) new List[params.length];
|
||||||
|
for (int i = 0; i < params.length; i++) {
|
||||||
|
paramSuper[i] = ClassUtil.getSuperClasses(params[i]);
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(" SUPER[" + params[i].getSimpleName() + "] => " + paramSuper[i]);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
|
||||||
|
for (Constructor<?> c : targetClass.getConstructors()) {
|
||||||
|
Class<?>[] ctypes = c.getParameterTypes();
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("CANDIDATE: " + c);
|
||||||
|
LOG.debug("CANDIDATE PARAMS: " + Arrays.toString(ctypes));
|
||||||
|
}
|
||||||
|
if (params.length != ctypes.length) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < params.length; i++) {
|
||||||
|
List<Class<?>> csuper = ClassUtil.getSuperClasses(ctypes[i]);
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(" SUPER[" + ctypes[i].getSimpleName() + "] => " + csuper);
|
||||||
|
}
|
||||||
|
if (CollectionUtils.intersection(paramSuper[i], csuper).isEmpty() == false) {
|
||||||
|
return ((Constructor<T>) c);
|
||||||
|
}
|
||||||
|
} // FOR (param)
|
||||||
|
} // FOR (constructors)
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Failed to retrieve constructor for " + targetClass.getSimpleName(), error);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param className
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static Class<?> getClass(String className) {
|
||||||
|
Class<?> targetClass = null;
|
||||||
|
try {
|
||||||
|
ClassLoader loader = ClassLoader.getSystemClassLoader();
|
||||||
|
targetClass = (Class<?>) loader.loadClass(className);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException("Failed to retrieve class for " + className, ex);
|
||||||
|
}
|
||||||
|
return (targetClass);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns true if asserts are enabled. This assumes that we're always using the default system
|
||||||
|
* ClassLoader
|
||||||
|
*/
|
||||||
|
public static boolean isAssertsEnabled() {
|
||||||
|
boolean ret = false;
|
||||||
|
try {
|
||||||
|
assert (false);
|
||||||
|
} catch (AssertionError ex) {
|
||||||
|
ret = true;
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,491 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - CollectionUtil.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import java.util.AbstractList;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Enumeration;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.NoSuchElementException;
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.SortedMap;
|
||||||
|
import org.apache.commons.collections15.set.ListOrderedSet;
|
||||||
|
import org.apache.commons.lang.NotImplementedException;
|
||||||
|
|
||||||
|
/** @author pavlo */
|
||||||
|
public abstract class CollectionUtil {
|
||||||
|
|
||||||
|
private static final Random RANDOM = new Random();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all of the elements in items into the given array This assumes that the array has been
|
||||||
|
* pre-allocated
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @param array
|
||||||
|
*/
|
||||||
|
public static <T> void toArray(
|
||||||
|
Collection<T> items, Object[] array, boolean convertToPrimitive) {
|
||||||
|
assert (items.size() == array.length);
|
||||||
|
|
||||||
|
int i = 0;
|
||||||
|
for (T t : items) {
|
||||||
|
if (convertToPrimitive) {
|
||||||
|
if (t instanceof Long) {
|
||||||
|
array[i] = ((Long) t).longValue();
|
||||||
|
} else if (t instanceof Integer) {
|
||||||
|
array[i] = ((Integer) t).intValue();
|
||||||
|
} else if (t instanceof Double) {
|
||||||
|
array[i] = ((Double) t).doubleValue();
|
||||||
|
} else if (t instanceof Boolean) {
|
||||||
|
array[i] = ((Boolean) t).booleanValue();
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
array[i] = t;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static int[] toIntArray(Collection<Integer> items) {
|
||||||
|
int[] ret = new int[items.size()];
|
||||||
|
int idx = 0;
|
||||||
|
for (Integer i : items) {
|
||||||
|
assert (i != null);
|
||||||
|
ret[idx++] = i.intValue();
|
||||||
|
} // FOR
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all the values of an Iterator into a List
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> List<T> list(Iterator<T> it) {
|
||||||
|
List<T> list = new ArrayList<T>();
|
||||||
|
CollectionUtil.addAll(list, it);
|
||||||
|
return (list);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all of the values of an Enumeration into a new List
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param e
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> List<T> list(Enumeration<T> e) {
|
||||||
|
return (list(iterable(e)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all of the values of an Iterable into a new List
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> List<T> list(Iterable<T> it) {
|
||||||
|
return (list(it.iterator()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all the values of an Iterator into a Set
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> Set<T> set(Iterator<T> it) {
|
||||||
|
Set<T> set = new HashSet<T>();
|
||||||
|
CollectionUtil.addAll(set, it);
|
||||||
|
return (set);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put all of the values of an Iterable into a new Set
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> Set<T> set(Iterable<T> it) {
|
||||||
|
return (set(it.iterator()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a list containing the string representations of the elements in the collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> List<String> toStringList(Collection<T> data) {
|
||||||
|
List<String> ret = new ArrayList<String>();
|
||||||
|
for (T t : data) {
|
||||||
|
ret.add(t.toString());
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a set containing the string representations of the elements in the collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> Set<String> toStringSet(Collection<T> data) {
|
||||||
|
Set<String> ret = new HashSet<String>();
|
||||||
|
for (T t : data) {
|
||||||
|
ret.add(t.toString());
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a random value from the given Collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
*/
|
||||||
|
public static <T> T random(Collection<T> items) {
|
||||||
|
return (CollectionUtil.random(items, RANDOM));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a random value from the given Collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @param rand
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T random(Collection<T> items, Random rand) {
|
||||||
|
int idx = rand.nextInt(items.size());
|
||||||
|
return (CollectionUtil.get(items, idx));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a random value from the given Iterable
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T random(Iterable<T> it) {
|
||||||
|
return (CollectionUtil.random(it, RANDOM));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a random value from the given Iterable
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @param rand
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T random(Iterable<T> it, Random rand) {
|
||||||
|
List<T> list = new ArrayList<T>();
|
||||||
|
for (T t : it) {
|
||||||
|
list.add(t);
|
||||||
|
} // FOR
|
||||||
|
return (CollectionUtil.random(list, rand));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <E extends Enum<?>> Set<E> getAllExcluding(E[] elements, E... excluding) {
|
||||||
|
Set<E> excludeSet = new HashSet<E>();
|
||||||
|
for (E e : excluding) {
|
||||||
|
excludeSet.add(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
Set<E> elementsSet = new HashSet<E>();
|
||||||
|
for (int i = 0; i < elements.length; i++) {
|
||||||
|
if (!excludeSet.contains(elements[i])) {
|
||||||
|
elementsSet.add(elements[i]);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
return (elementsSet);
|
||||||
|
// Crappy java....
|
||||||
|
// Object new_elements[] = new Object[elements_set.size()];
|
||||||
|
// elements_set.toArray(new_elements);
|
||||||
|
// return ((E[])new_elements);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add all the items in the array to a Collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @param items
|
||||||
|
*/
|
||||||
|
public static <T> Collection<T> addAll(Collection<T> data, T... items) {
|
||||||
|
for (T i : (T[]) items) {
|
||||||
|
data.add(i);
|
||||||
|
}
|
||||||
|
return (data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add all the items in the Enumeration into a Collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @param items
|
||||||
|
*/
|
||||||
|
public static <T> Collection<T> addAll(Collection<T> data, Enumeration<T> items) {
|
||||||
|
while (items.hasMoreElements()) {
|
||||||
|
data.add(items.nextElement());
|
||||||
|
} // WHILE
|
||||||
|
return (data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add all of the items from the Iterable into the given collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @param items
|
||||||
|
*/
|
||||||
|
public static <T> Collection<T> addAll(Collection<T> data, Iterable<T> items) {
|
||||||
|
return (CollectionUtil.addAll(data, items.iterator()));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add all of the items from the Iterator into the given collection
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param data
|
||||||
|
* @param items
|
||||||
|
*/
|
||||||
|
public static <T> Collection<T> addAll(Collection<T> data, Iterator<T> items) {
|
||||||
|
while (items.hasNext()) {
|
||||||
|
data.add(items.next());
|
||||||
|
} // WHILE
|
||||||
|
return (data);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param <T>
|
||||||
|
* @param <U>
|
||||||
|
* @param map
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T, U extends Comparable<U>> T getGreatest(Map<T, U> map) {
|
||||||
|
T maxKey = null;
|
||||||
|
U maxValue = null;
|
||||||
|
for (Entry<T, U> e : map.entrySet()) {
|
||||||
|
T key = e.getKey();
|
||||||
|
U value = e.getValue();
|
||||||
|
if (maxValue == null || value.compareTo(maxValue) > 0) {
|
||||||
|
maxValue = value;
|
||||||
|
maxKey = key;
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
return (maxKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the first item in a Iterable
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T first(Iterable<T> items) {
|
||||||
|
return (CollectionUtil.get(items, 0));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the first item in a Iterator
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T first(Iterator<T> items) {
|
||||||
|
return (items.hasNext() ? items.next() : null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the first item in an Enumeration
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T first(Enumeration<T> items) {
|
||||||
|
return (items.hasMoreElements() ? items.nextElement() : null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the ith element of a set. Super lame
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @param idx
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T get(Iterable<T> items, int idx) {
|
||||||
|
if (items instanceof AbstractList<?>) {
|
||||||
|
return ((AbstractList<T>) items).get(idx);
|
||||||
|
} else if (items instanceof ListOrderedSet<?>) {
|
||||||
|
return ((ListOrderedSet<T>) items).get(idx);
|
||||||
|
}
|
||||||
|
int ctr = 0;
|
||||||
|
for (T t : items) {
|
||||||
|
if (ctr++ == idx) {
|
||||||
|
return (t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return (null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the last item in an Iterable
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T last(Iterable<T> items) {
|
||||||
|
T last = null;
|
||||||
|
if (items instanceof AbstractList<?>) {
|
||||||
|
AbstractList<T> list = (AbstractList<T>) items;
|
||||||
|
last = (list.isEmpty() ? null : list.get(list.size() - 1));
|
||||||
|
} else {
|
||||||
|
for (T t : items) {
|
||||||
|
last = t;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return (last);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the last item in an array
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param items
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> T last(T... items) {
|
||||||
|
if (items != null && items.length > 0) {
|
||||||
|
return (items[items.length - 1]);
|
||||||
|
}
|
||||||
|
return (null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param <K>
|
||||||
|
* @param <V>
|
||||||
|
* @param map
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <K extends Comparable<?>, V> List<V> getSortedList(
|
||||||
|
SortedMap<K, Collection<V>> map) {
|
||||||
|
List<V> ret = new ArrayList<V>();
|
||||||
|
for (Collection<V> col : map.values()) {
|
||||||
|
ret.addAll(col);
|
||||||
|
} // FOR
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap an Iterable around an Iterator
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param it
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> Iterable<T> iterable(final Iterator<T> it) {
|
||||||
|
return (new Iterable<T>() {
|
||||||
|
@Override
|
||||||
|
public Iterator<T> iterator() {
|
||||||
|
return (it);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> Iterable<T> iterable(final T[] values) {
|
||||||
|
return (new Iterable<T>() {
|
||||||
|
@Override
|
||||||
|
public Iterator<T> iterator() {
|
||||||
|
return new Iterator<T>() {
|
||||||
|
private int idx = 0;
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return (this.idx < values.length);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public T next() {
|
||||||
|
if (this.idx == values.length) {
|
||||||
|
throw new NoSuchElementException();
|
||||||
|
}
|
||||||
|
return values[this.idx++];
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
throw new UnsupportedOperationException();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wrap an Iterable around an Enumeration
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param e
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T> Iterable<T> iterable(final Enumeration<T> e) {
|
||||||
|
return (new Iterable<T>() {
|
||||||
|
@Override
|
||||||
|
public Iterator<T> iterator() {
|
||||||
|
return new Iterator<T>() {
|
||||||
|
@Override
|
||||||
|
public boolean hasNext() {
|
||||||
|
return (e.hasMoreElements());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public T next() {
|
||||||
|
return (e.nextElement());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void remove() {
|
||||||
|
throw new NotImplementedException();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T pop(Collection<T> items) {
|
||||||
|
T t = CollectionUtil.first(items);
|
||||||
|
if (t != null) {
|
||||||
|
boolean ret = items.remove(t);
|
||||||
|
assert (ret);
|
||||||
|
}
|
||||||
|
return (t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,383 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - FileUtil.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.FileWriter;
|
||||||
|
import java.io.FilenameFilter;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStreamReader;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
import java.util.zip.GZIPInputStream;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
/** @author pavlo */
|
||||||
|
public abstract class FileUtil {
|
||||||
|
private static final Logger LOG = Logger.getLogger(FileUtil.class);
|
||||||
|
|
||||||
|
private static final Pattern EXT_SPLIT = Pattern.compile("\\.");
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Join path components
|
||||||
|
*
|
||||||
|
* @param args
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static String joinPath(String... args) {
|
||||||
|
StringBuilder result = new StringBuilder();
|
||||||
|
boolean first = true;
|
||||||
|
for (String a : args) {
|
||||||
|
if (a != null && a.length() > 0) {
|
||||||
|
if (!first) {
|
||||||
|
result.append("/");
|
||||||
|
}
|
||||||
|
result.append(a);
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given a basename for a file, find the next possible filename if this file already exists. For
|
||||||
|
* example, if the file test.res already exists, create a file called, test.1.res
|
||||||
|
*
|
||||||
|
* @param basename
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static String getNextFilename(String basename) {
|
||||||
|
|
||||||
|
if (!exists(basename)) {
|
||||||
|
return basename;
|
||||||
|
}
|
||||||
|
|
||||||
|
File f = new File(basename);
|
||||||
|
if (f != null && f.isFile()) {
|
||||||
|
String[] parts = EXT_SPLIT.split(basename);
|
||||||
|
|
||||||
|
// Check how many files already exist
|
||||||
|
int counter = 1;
|
||||||
|
String nextName = parts[0] + "." + counter + "." + parts[1];
|
||||||
|
while (exists(nextName)) {
|
||||||
|
++counter;
|
||||||
|
nextName = parts[0] + "." + counter + "." + parts[1];
|
||||||
|
}
|
||||||
|
return nextName;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Should we throw instead??
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static boolean exists(String path) {
|
||||||
|
return (new File(path).exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String realpath(String path) {
|
||||||
|
File f = new File(path);
|
||||||
|
String ret = null;
|
||||||
|
try {
|
||||||
|
ret = f.getCanonicalPath();
|
||||||
|
} catch (Exception ex) {
|
||||||
|
LOG.warn(ex);
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String basename(String path) {
|
||||||
|
return (new File(path)).getName();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getExtension(File f) {
|
||||||
|
if (f != null && f.isFile()) {
|
||||||
|
String[] parts = EXT_SPLIT.split(f.getName());
|
||||||
|
if (parts.length > 1) {
|
||||||
|
return (parts[parts.length - 1]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return (null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create any directory in the list paths if it doesn't exist
|
||||||
|
*
|
||||||
|
* @param paths
|
||||||
|
*/
|
||||||
|
public static void makeDirIfNotExists(String... paths) {
|
||||||
|
for (String p : paths) {
|
||||||
|
if (p == null) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
File f = new File(p);
|
||||||
|
if (f.exists() == false) {
|
||||||
|
f.mkdirs();
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return a File handle to a temporary file location
|
||||||
|
*
|
||||||
|
* @param ext the suffix of the filename
|
||||||
|
* @param deleteOnExit whether to delete this file after the JVM exits
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static File getTempFile(String ext, boolean deleteOnExit) {
|
||||||
|
return getTempFile(null, ext, deleteOnExit);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static File getTempFile(String ext) {
|
||||||
|
return (FileUtil.getTempFile(null, ext, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static File getTempFile(String prefix, String suffix, boolean deleteOnExit) {
|
||||||
|
File tempFile;
|
||||||
|
if (suffix != null && suffix.startsWith(".") == false) {
|
||||||
|
suffix = "." + suffix;
|
||||||
|
}
|
||||||
|
if (prefix == null) {
|
||||||
|
prefix = "hstore";
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
tempFile = File.createTempFile(prefix, suffix);
|
||||||
|
if (deleteOnExit) {
|
||||||
|
tempFile.deleteOnExit();
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
return (tempFile);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Unsafely create a temporary directory Yes I said that this was unsafe. I don't care...
|
||||||
|
*
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static File getTempDirectory() {
|
||||||
|
final File temp = FileUtil.getTempFile(null);
|
||||||
|
if (!(temp.delete())) {
|
||||||
|
throw new RuntimeException("Could not delete temp file: " + temp.getAbsolutePath());
|
||||||
|
} else if (!(temp.mkdir())) {
|
||||||
|
throw new RuntimeException("Could not create temp directory: " + temp.getAbsolutePath());
|
||||||
|
}
|
||||||
|
return (temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static File writeStringToFile(String filePath, String content) throws IOException {
|
||||||
|
return (FileUtil.writeStringToFile(new File(filePath), content));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static File writeStringToFile(File file, String content) throws IOException {
|
||||||
|
FileWriter writer = new FileWriter(file);
|
||||||
|
writer.write(content);
|
||||||
|
writer.flush();
|
||||||
|
writer.close();
|
||||||
|
return (file);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the given string to a temporary file Will not delete the file after the JVM exits
|
||||||
|
*
|
||||||
|
* @param content
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static File writeStringToTempFile(String content) {
|
||||||
|
return (writeStringToTempFile(content, "tmp", false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the given string to a temporary file with the given extension as the suffix Will not
|
||||||
|
* delete the file after the JVM exits
|
||||||
|
*
|
||||||
|
* @param content
|
||||||
|
* @param ext
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static File writeStringToTempFile(String content, String ext) {
|
||||||
|
return (writeStringToTempFile(content, ext, false));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the given string to a temporary file with the given extension as the suffix If
|
||||||
|
* deleteOnExit is true, then the file will be removed when the JVM exits
|
||||||
|
*
|
||||||
|
* @param content
|
||||||
|
* @param ext
|
||||||
|
* @param deleteOnExit
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static File writeStringToTempFile(String content, String ext, boolean deleteOnExit) {
|
||||||
|
File tempFile = FileUtil.getTempFile(ext, deleteOnExit);
|
||||||
|
try {
|
||||||
|
FileUtil.writeStringToFile(tempFile, content);
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
return tempFile;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String readFile(File path) {
|
||||||
|
LOG.debug(path);
|
||||||
|
return (readFile(path.getAbsolutePath()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String readFile(String path) {
|
||||||
|
StringBuilder buffer = new StringBuilder();
|
||||||
|
try {
|
||||||
|
BufferedReader in = FileUtil.getReader(path);
|
||||||
|
while (in.ready()) {
|
||||||
|
buffer.append(in.readLine()).append("\n");
|
||||||
|
} // WHILE
|
||||||
|
in.close();
|
||||||
|
} catch (IOException ex) {
|
||||||
|
throw new RuntimeException("Failed to read file contents from '" + path + "'", ex);
|
||||||
|
}
|
||||||
|
return (buffer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a BufferedReader for the given input path Can handle both gzip and plain text files
|
||||||
|
*
|
||||||
|
* @param path
|
||||||
|
* @return
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static BufferedReader getReader(String path) throws IOException {
|
||||||
|
return (FileUtil.getReader(new File(path)));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Creates a BufferedReader for the given input path Can handle both gzip and plain text files
|
||||||
|
*
|
||||||
|
* @param file
|
||||||
|
* @return
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static BufferedReader getReader(File file) throws IOException {
|
||||||
|
if (!file.exists()) {
|
||||||
|
throw new IOException("The file '" + file + "' does not exist");
|
||||||
|
}
|
||||||
|
|
||||||
|
BufferedReader in = null;
|
||||||
|
if (file.getPath().endsWith(".gz")) {
|
||||||
|
FileInputStream fin = new FileInputStream(file);
|
||||||
|
GZIPInputStream gzis = new GZIPInputStream(fin);
|
||||||
|
in = new BufferedReader(new InputStreamReader(gzis));
|
||||||
|
LOG.debug("Reading in the zipped contents of '" + file.getName() + "'");
|
||||||
|
} else {
|
||||||
|
in = new BufferedReader(new FileReader(file));
|
||||||
|
LOG.debug("Reading in the contents of '" + file.getName() + "'");
|
||||||
|
}
|
||||||
|
return (in);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static byte[] readBytesFromFile(String path) throws IOException {
|
||||||
|
File file = new File(path);
|
||||||
|
FileInputStream in = new FileInputStream(file);
|
||||||
|
|
||||||
|
// Create the byte array to hold the data
|
||||||
|
long length = file.length();
|
||||||
|
byte[] bytes = new byte[(int) length];
|
||||||
|
|
||||||
|
LOG.debug("Reading in the contents of '" + file.getAbsolutePath() + "'");
|
||||||
|
|
||||||
|
// Read in the bytes
|
||||||
|
int offset = 0;
|
||||||
|
int numRead = 0;
|
||||||
|
while ((offset < bytes.length)
|
||||||
|
&& ((numRead = in.read(bytes, offset, bytes.length - offset)) >= 0)) {
|
||||||
|
offset += numRead;
|
||||||
|
} // WHILE
|
||||||
|
if (offset < bytes.length) {
|
||||||
|
throw new IOException("Failed to read the entire contents of '" + file.getName() + "'");
|
||||||
|
}
|
||||||
|
in.close();
|
||||||
|
return (bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the path to a directory below our current location in the source tree Throws a
|
||||||
|
* RuntimeException if we go beyond our repository checkout
|
||||||
|
*
|
||||||
|
* @param dirName
|
||||||
|
* @return
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static File findDirectory(String dirName) throws IOException {
|
||||||
|
return (FileUtil.find(dirName, new File(".").getCanonicalFile(), true).getCanonicalFile());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Find the path to a directory below our current location in the source tree Throws a
|
||||||
|
* RuntimeException if we go beyond our repository checkout
|
||||||
|
*
|
||||||
|
* @param dirName
|
||||||
|
* @return
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static File findFile(String fileName) throws IOException {
|
||||||
|
return (FileUtil.find(fileName, new File(".").getCanonicalFile(), false).getCanonicalFile());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static final File find(String name, File current, boolean isdir) throws IOException {
|
||||||
|
LOG.debug("Find Current Location = " + current);
|
||||||
|
boolean hasSvn = false;
|
||||||
|
for (File file : current.listFiles()) {
|
||||||
|
if (file.getCanonicalPath().endsWith(File.separator + name) && file.isDirectory() == isdir) {
|
||||||
|
return (file);
|
||||||
|
// Make sure that we don't go to far down...
|
||||||
|
} else if (file.getCanonicalPath().endsWith(File.separator + ".svn")) {
|
||||||
|
hasSvn = true;
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
// If we didn't see an .svn directory, then we went too far down
|
||||||
|
if (!hasSvn) {
|
||||||
|
throw new RuntimeException(
|
||||||
|
"Unable to find directory '" + name + "' [last_dir=" + current.getAbsolutePath() + "]");
|
||||||
|
}
|
||||||
|
File next = new File(current.getCanonicalPath() + File.separator + "");
|
||||||
|
return (FileUtil.find(name, next, isdir));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns a list of all the files in a directory whose name starts with the provided prefix
|
||||||
|
*
|
||||||
|
* @param dir
|
||||||
|
* @param filePrefix
|
||||||
|
* @return
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static List<File> getFilesInDirectory(final File dir, final String filePrefix)
|
||||||
|
throws IOException {
|
||||||
|
assert (dir.isDirectory()) : "Invalid search directory path: " + dir;
|
||||||
|
FilenameFilter filter =
|
||||||
|
new FilenameFilter() {
|
||||||
|
@Override
|
||||||
|
public boolean accept(File dir, String name) {
|
||||||
|
return (name.startsWith(filePrefix));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return (Arrays.asList(dir.listFiles(filter)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String getAbsPath(String dirPath) {
|
||||||
|
String current = null;
|
||||||
|
try {
|
||||||
|
current = new java.io.File(dirPath).getCanonicalPath();
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - JSONSerializable.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONString;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.io.IOException;
|
||||||
|
|
||||||
|
public interface JSONSerializable extends JSONString {
|
||||||
|
public void save(String outputPath) throws IOException;
|
||||||
|
|
||||||
|
public void load(String inputPath) throws IOException;
|
||||||
|
|
||||||
|
public void toJSON(JSONStringer stringer) throws JSONException;
|
||||||
|
|
||||||
|
public void fromJSON(JSONObject jsonObject) throws JSONException;
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,800 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - JSONUtil.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import com.controller.util.json.JSONArray;
|
||||||
|
import com.controller.util.json.JSONException;
|
||||||
|
import com.controller.util.json.JSONObject;
|
||||||
|
import com.controller.util.json.JSONStringer;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.lang.reflect.Field;
|
||||||
|
import java.lang.reflect.Modifier;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Collections;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Map.Entry;
|
||||||
|
import java.util.Set;
|
||||||
|
import java.util.Stack;
|
||||||
|
import org.apache.log4j.Logger;
|
||||||
|
|
||||||
|
/** @author pavlo */
|
||||||
|
public abstract class JSONUtil {
|
||||||
|
private static final Logger LOG = Logger.getLogger(JSONUtil.class.getName());
|
||||||
|
|
||||||
|
private static final String JSON_CLASS_SUFFIX = "_class";
|
||||||
|
private static final Map<Class<?>, Field[]> SERIALIZABLE_FIELDS =
|
||||||
|
new HashMap<Class<?>, Field[]>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param clazz
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static Field[] getSerializableFields(Class<?> clazz, String... fieldsToExclude) {
|
||||||
|
Field[] ret = SERIALIZABLE_FIELDS.get(clazz);
|
||||||
|
if (ret == null) {
|
||||||
|
Collection<String> exclude = CollectionUtil.addAll(new HashSet<String>(), fieldsToExclude);
|
||||||
|
synchronized (SERIALIZABLE_FIELDS) {
|
||||||
|
ret = SERIALIZABLE_FIELDS.get(clazz);
|
||||||
|
if (ret == null) {
|
||||||
|
List<Field> fields = new ArrayList<Field>();
|
||||||
|
for (Field f : clazz.getFields()) {
|
||||||
|
int modifiers = f.getModifiers();
|
||||||
|
if (Modifier.isTransient(modifiers) == false
|
||||||
|
&& Modifier.isPublic(modifiers) == true
|
||||||
|
&& Modifier.isStatic(modifiers) == false
|
||||||
|
&& exclude.contains(f.getName()) == false) {
|
||||||
|
fields.add(f);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
ret = fields.toArray(new Field[0]);
|
||||||
|
SERIALIZABLE_FIELDS.put(clazz, ret);
|
||||||
|
}
|
||||||
|
} // SYNCH
|
||||||
|
}
|
||||||
|
return (ret);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON Pretty Print
|
||||||
|
*
|
||||||
|
* @param json
|
||||||
|
* @return
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static String format(String json) {
|
||||||
|
try {
|
||||||
|
return (JSONUtil.format(new JSONObject(json)));
|
||||||
|
} catch (RuntimeException ex) {
|
||||||
|
throw ex;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSON Pretty Print
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param object
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static <T extends JSONSerializable> String format(T object) {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
if (object instanceof JSONObject) {
|
||||||
|
return ((JSONObject) object).toString(2);
|
||||||
|
}
|
||||||
|
stringer.object();
|
||||||
|
object.toJSON(stringer);
|
||||||
|
stringer.endObject();
|
||||||
|
} catch (JSONException ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
return (JSONUtil.format(stringer.toString()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String format(JSONObject o) {
|
||||||
|
try {
|
||||||
|
return o.toString(1);
|
||||||
|
} catch (JSONException ex) {
|
||||||
|
throw new RuntimeException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param <T>
|
||||||
|
* @param object
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
public static String toJSONString(Object object) {
|
||||||
|
JSONStringer stringer = new JSONStringer();
|
||||||
|
try {
|
||||||
|
if (object instanceof JSONSerializable) {
|
||||||
|
stringer.object();
|
||||||
|
((JSONSerializable) object).toJSON(stringer);
|
||||||
|
stringer.endObject();
|
||||||
|
} else if (object != null) {
|
||||||
|
Class<?> clazz = object.getClass();
|
||||||
|
// stringer.key(clazz.getSimpleName());
|
||||||
|
JSONUtil.writeFieldValue(stringer, clazz, object);
|
||||||
|
}
|
||||||
|
} catch (JSONException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
return (stringer.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T extends JSONSerializable> T fromJSONString(T t, String json) {
|
||||||
|
try {
|
||||||
|
JSONObject jsonObject = new JSONObject(json);
|
||||||
|
t.fromJSON(jsonObject);
|
||||||
|
} catch (JSONException ex) {
|
||||||
|
throw new RuntimeException("Failed to deserialize object " + t, ex);
|
||||||
|
}
|
||||||
|
return (t);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the contents of a JSONSerializable object out to a file on the local disk
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param object
|
||||||
|
* @param outputPath
|
||||||
|
* @throws IOException
|
||||||
|
*/
|
||||||
|
public static <T extends JSONSerializable> void save(T object, String outputPath)
|
||||||
|
throws IOException {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(
|
||||||
|
"Writing out contents of "
|
||||||
|
+ object.getClass().getSimpleName()
|
||||||
|
+ " to '"
|
||||||
|
+ outputPath
|
||||||
|
+ "'");
|
||||||
|
}
|
||||||
|
File f = new File(outputPath);
|
||||||
|
try {
|
||||||
|
FileUtil.makeDirIfNotExists(f.getParent());
|
||||||
|
String json = object.toJSONString();
|
||||||
|
FileUtil.writeStringToFile(f, format(json));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
LOG.error(
|
||||||
|
"Failed to serialize the " + object.getClass().getSimpleName() + " file '" + f + "'", ex);
|
||||||
|
throw new IOException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Load in a JSONSerialable stored in a file
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param object
|
||||||
|
* @param inputPath
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public static <T extends JSONSerializable> void load(T object, String inputPath)
|
||||||
|
throws IOException {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(
|
||||||
|
"Loading in serialized "
|
||||||
|
+ object.getClass().getSimpleName()
|
||||||
|
+ " from '"
|
||||||
|
+ inputPath
|
||||||
|
+ "'");
|
||||||
|
}
|
||||||
|
String contents = FileUtil.readFile(inputPath);
|
||||||
|
if (contents.isEmpty()) {
|
||||||
|
throw new IOException(
|
||||||
|
"The " + object.getClass().getSimpleName() + " file '" + inputPath + "' is empty");
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
object.fromJSON(new JSONObject(contents));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.error(
|
||||||
|
"Failed to deserialize the "
|
||||||
|
+ object.getClass().getSimpleName()
|
||||||
|
+ " from file '"
|
||||||
|
+ inputPath
|
||||||
|
+ "'",
|
||||||
|
ex);
|
||||||
|
}
|
||||||
|
throw new IOException(ex);
|
||||||
|
}
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("The loading of the " + object.getClass().getSimpleName() + " is complete");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For a given Enum, write out the contents of the corresponding field to the JSONObject We assume
|
||||||
|
* that the given object has matching fields that correspond to the Enum members, except that
|
||||||
|
* their names are lower case.
|
||||||
|
*
|
||||||
|
* @param <E>
|
||||||
|
* @param <T>
|
||||||
|
* @param stringer
|
||||||
|
* @param object
|
||||||
|
* @param baseClass
|
||||||
|
* @param members
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static <E extends Enum<?>, T> void fieldsToJSON(
|
||||||
|
JSONStringer stringer, T object, Class<? extends T> baseClass, E[] members)
|
||||||
|
throws JSONException {
|
||||||
|
try {
|
||||||
|
fieldsToJSON(
|
||||||
|
stringer, object, baseClass, ClassUtil.getFieldsFromMembersEnum(baseClass, members));
|
||||||
|
} catch (NoSuchFieldException ex) {
|
||||||
|
throw new JSONException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For a given list of Fields, write out the contents of the corresponding field to the JSONObject
|
||||||
|
* The each of the JSONObject's elements will be the upper case version of the Field's name
|
||||||
|
*
|
||||||
|
* @param <T>
|
||||||
|
* @param stringer
|
||||||
|
* @param object
|
||||||
|
* @param baseClass
|
||||||
|
* @param fields
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static <T> void fieldsToJSON(
|
||||||
|
JSONStringer stringer, T object, Class<? extends T> baseClass, Field[] fields)
|
||||||
|
throws JSONException {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Serializing out " + fields.length + " elements for " + baseClass.getSimpleName());
|
||||||
|
}
|
||||||
|
for (Field f : fields) {
|
||||||
|
String jsonKey = f.getName().toUpperCase();
|
||||||
|
stringer.key(jsonKey);
|
||||||
|
|
||||||
|
try {
|
||||||
|
Class<?> fclass = f.getType();
|
||||||
|
Object fvalue = f.get(object);
|
||||||
|
|
||||||
|
// Null
|
||||||
|
if (fvalue == null) {
|
||||||
|
writeFieldValue(stringer, fclass, fvalue);
|
||||||
|
// Maps
|
||||||
|
} else if (fvalue instanceof Map) {
|
||||||
|
writeFieldValue(stringer, fclass, fvalue);
|
||||||
|
// Everything else
|
||||||
|
} else {
|
||||||
|
writeFieldValue(stringer, fclass, fvalue);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
throw new JSONException(ex);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param stringer
|
||||||
|
* @param fieldClass
|
||||||
|
* @param fieldValue
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static void writeFieldValue(
|
||||||
|
JSONStringer stringer, Class<?> fieldClass, Object fieldValue) throws JSONException {
|
||||||
|
// Null
|
||||||
|
if (fieldValue == null) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("writeNullFieldValue(" + fieldClass + ", " + fieldValue + ")");
|
||||||
|
}
|
||||||
|
stringer.value(null);
|
||||||
|
|
||||||
|
// Collections
|
||||||
|
} else if (ClassUtil.getInterfaces(fieldClass).contains(Collection.class)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("writeCollectionFieldValue(" + fieldClass + ", " + fieldValue + ")");
|
||||||
|
}
|
||||||
|
stringer.array();
|
||||||
|
for (Object value : (Collection<?>) fieldValue) {
|
||||||
|
if (value == null) {
|
||||||
|
stringer.value(null);
|
||||||
|
} else {
|
||||||
|
writeFieldValue(stringer, value.getClass(), value);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
stringer.endArray();
|
||||||
|
|
||||||
|
// Maps
|
||||||
|
} else if (fieldValue instanceof Map) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("writeMapFieldValue(" + fieldClass + ", " + fieldValue + ")");
|
||||||
|
}
|
||||||
|
stringer.object();
|
||||||
|
for (Entry<?, ?> e : ((Map<?, ?>) fieldValue).entrySet()) {
|
||||||
|
// We can handle null keys
|
||||||
|
String keyValue = null;
|
||||||
|
if (e.getKey() != null) {
|
||||||
|
// deserialize it on the other side
|
||||||
|
Class<?> keyClass = e.getKey().getClass();
|
||||||
|
keyValue = makePrimitiveValue(keyClass, e.getKey()).toString();
|
||||||
|
}
|
||||||
|
stringer.key(keyValue);
|
||||||
|
|
||||||
|
// We can also handle null values. Where is your god now???
|
||||||
|
if (e.getValue() == null) {
|
||||||
|
stringer.value(null);
|
||||||
|
} else {
|
||||||
|
writeFieldValue(stringer, e.getValue().getClass(), e.getValue());
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
stringer.endObject();
|
||||||
|
|
||||||
|
// Primitive
|
||||||
|
} else {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("writePrimitiveFieldValue(" + fieldClass + ", " + fieldValue + ")");
|
||||||
|
}
|
||||||
|
stringer.value(makePrimitiveValue(fieldClass, fieldValue));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read data from the given JSONObject and populate the given Map
|
||||||
|
*
|
||||||
|
* @param jsonObject
|
||||||
|
* @param map
|
||||||
|
* @param innerClasses
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
protected static void readMapField(
|
||||||
|
final JSONObject jsonObject, final Map map, final Stack<Class> innerClasses)
|
||||||
|
throws Exception {
|
||||||
|
Class<?> keyClass = innerClasses.pop();
|
||||||
|
Class<?> valClass = innerClasses.pop();
|
||||||
|
Collection<Class<?>> valInterfaces = ClassUtil.getInterfaces(valClass);
|
||||||
|
|
||||||
|
assert (jsonObject != null);
|
||||||
|
for (String jsonKey : CollectionUtil.iterable(jsonObject.keys())) {
|
||||||
|
final Stack<Class> nextInnerClasses = new Stack<Class>();
|
||||||
|
nextInnerClasses.addAll(innerClasses);
|
||||||
|
assert (nextInnerClasses.equals(innerClasses));
|
||||||
|
|
||||||
|
// KEY
|
||||||
|
Object key = JSONUtil.getPrimitiveValue(jsonKey, keyClass);
|
||||||
|
|
||||||
|
// VALUE
|
||||||
|
Object object = null;
|
||||||
|
if (jsonObject.isNull(jsonKey)) {
|
||||||
|
// Nothing...
|
||||||
|
} else if (valInterfaces.contains(List.class)) {
|
||||||
|
object = new ArrayList();
|
||||||
|
readCollectionField(
|
||||||
|
jsonObject.getJSONArray(jsonKey), (Collection) object, nextInnerClasses);
|
||||||
|
} else if (valInterfaces.contains(Set.class)) {
|
||||||
|
object = new HashSet();
|
||||||
|
readCollectionField(
|
||||||
|
jsonObject.getJSONArray(jsonKey), (Collection) object, nextInnerClasses);
|
||||||
|
} else if (valInterfaces.contains(Map.class)) {
|
||||||
|
object = new HashMap();
|
||||||
|
readMapField(jsonObject.getJSONObject(jsonKey), (Map) object, nextInnerClasses);
|
||||||
|
} else {
|
||||||
|
String jsonString = jsonObject.getString(jsonKey);
|
||||||
|
try {
|
||||||
|
object = JSONUtil.getPrimitiveValue(jsonString, valClass);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
System.err.println("val_interfaces: " + valInterfaces);
|
||||||
|
LOG.error(
|
||||||
|
"Failed to deserialize value '"
|
||||||
|
+ jsonString
|
||||||
|
+ "' from inner map key '"
|
||||||
|
+ jsonKey
|
||||||
|
+ "'");
|
||||||
|
throw ex;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
map.put(key, object);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read data from the given JSONArray and populate the given Collection
|
||||||
|
*
|
||||||
|
* @param jsonArray
|
||||||
|
* @param collection
|
||||||
|
* @param innerClasses
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
protected static void readCollectionField(
|
||||||
|
final JSONArray jsonArray, final Collection collection, final Stack<Class> innerClasses)
|
||||||
|
throws Exception {
|
||||||
|
// We need to figure out what the inner type of the collection is
|
||||||
|
// If it's a Collection or a Map, then we need to instantiate it before
|
||||||
|
// we can call readFieldValue() again for it.
|
||||||
|
Class innerClass = innerClasses.pop();
|
||||||
|
Collection<Class<?>> innerInterfaces = ClassUtil.getInterfaces(innerClass);
|
||||||
|
|
||||||
|
for (int i = 0, cnt = jsonArray.length(); i < cnt; i++) {
|
||||||
|
final Stack<Class> nextInnerClasses = new Stack<Class>();
|
||||||
|
nextInnerClasses.addAll(innerClasses);
|
||||||
|
assert (nextInnerClasses.equals(innerClasses));
|
||||||
|
Object value = null;
|
||||||
|
|
||||||
|
// Null
|
||||||
|
if (jsonArray.isNull(i)) {
|
||||||
|
value = null;
|
||||||
|
// Lists
|
||||||
|
} else if (innerInterfaces.contains(List.class)) {
|
||||||
|
value = new ArrayList();
|
||||||
|
readCollectionField(jsonArray.getJSONArray(i), (Collection) value, nextInnerClasses);
|
||||||
|
// Sets
|
||||||
|
} else if (innerInterfaces.contains(Set.class)) {
|
||||||
|
value = new HashSet();
|
||||||
|
readCollectionField(jsonArray.getJSONArray(i), (Collection) value, nextInnerClasses);
|
||||||
|
// Maps
|
||||||
|
} else if (innerInterfaces.contains(Map.class)) {
|
||||||
|
value = new HashMap();
|
||||||
|
readMapField(jsonArray.getJSONObject(i), (Map) value, nextInnerClasses);
|
||||||
|
// Values
|
||||||
|
} else {
|
||||||
|
String jsonString = jsonArray.getString(i);
|
||||||
|
value = JSONUtil.getPrimitiveValue(jsonString, innerClass);
|
||||||
|
}
|
||||||
|
collection.add(value);
|
||||||
|
} // FOR
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param jsonObject
|
||||||
|
* @param jsonKey
|
||||||
|
* @param fieldHandle
|
||||||
|
* @param object
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
public static void readFieldValue(
|
||||||
|
final JSONObject jsonObject, final String jsonKey, Field fieldHandle, Object object)
|
||||||
|
throws Exception {
|
||||||
|
assert (jsonObject.has(jsonKey)) : "No entry exists for '" + jsonKey + "'";
|
||||||
|
Class<?> fieldClass = fieldHandle.getType();
|
||||||
|
Object fieldObject = fieldHandle.get(object);
|
||||||
|
// String field_name = field_handle.getName();
|
||||||
|
|
||||||
|
// Null
|
||||||
|
if (jsonObject.isNull(jsonKey)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Field " + jsonKey + " is null");
|
||||||
|
}
|
||||||
|
fieldHandle.set(object, null);
|
||||||
|
|
||||||
|
// Collections
|
||||||
|
} else if (ClassUtil.getInterfaces(fieldClass).contains(Collection.class)) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Field " + jsonKey + " is a collection");
|
||||||
|
}
|
||||||
|
assert (fieldObject != null);
|
||||||
|
Stack<Class> innerClasses = new Stack<Class>();
|
||||||
|
innerClasses.addAll(ClassUtil.getGenericTypes(fieldHandle));
|
||||||
|
Collections.reverse(innerClasses);
|
||||||
|
|
||||||
|
JSONArray jsonInner = jsonObject.getJSONArray(jsonKey);
|
||||||
|
if (jsonInner == null) {
|
||||||
|
throw new JSONException("No array exists for '" + jsonKey + "'");
|
||||||
|
}
|
||||||
|
readCollectionField(jsonInner, (Collection) fieldObject, innerClasses);
|
||||||
|
|
||||||
|
// Maps
|
||||||
|
} else if (fieldObject instanceof Map) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Field " + jsonKey + " is a map");
|
||||||
|
}
|
||||||
|
assert (fieldObject != null);
|
||||||
|
Stack<Class> innerClasses = new Stack<Class>();
|
||||||
|
innerClasses.addAll(ClassUtil.getGenericTypes(fieldHandle));
|
||||||
|
Collections.reverse(innerClasses);
|
||||||
|
|
||||||
|
JSONObject jsonInner = jsonObject.getJSONObject(jsonKey);
|
||||||
|
if (jsonInner == null) {
|
||||||
|
throw new JSONException("No object exists for '" + jsonKey + "'");
|
||||||
|
}
|
||||||
|
readMapField(jsonInner, (Map) fieldObject, innerClasses);
|
||||||
|
|
||||||
|
// Everything else...
|
||||||
|
} else {
|
||||||
|
Class explicitFieldClass = JSONUtil.getClassForField(jsonObject, jsonKey);
|
||||||
|
if (explicitFieldClass != null) {
|
||||||
|
fieldClass = explicitFieldClass;
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug(
|
||||||
|
"Found explict field class " + fieldClass.getSimpleName() + " for " + jsonKey);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Field " + jsonKey + " is primitive type " + fieldClass.getSimpleName());
|
||||||
|
}
|
||||||
|
Object value = JSONUtil.getPrimitiveValue(jsonObject.getString(jsonKey), fieldClass);
|
||||||
|
fieldHandle.set(object, value);
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Set field " + jsonKey + " to '" + value + "'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For the given enum, load in the values from the JSON object into the current object This will
|
||||||
|
* throw errors if a field is missing
|
||||||
|
*
|
||||||
|
* @param <E>
|
||||||
|
* @param jsonObject
|
||||||
|
* @param members
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static <E extends Enum<?>, T> void fieldsFromJSON(
|
||||||
|
JSONObject jsonObject, T object, Class<? extends T> baseClass, E... members)
|
||||||
|
throws JSONException {
|
||||||
|
JSONUtil.fieldsFromJSON(jsonObject, object, baseClass, false, members);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For the given enum, load in the values from the JSON object into the current object If
|
||||||
|
* ignore_missing is false, then JSONUtil will not throw an error if a field is missing
|
||||||
|
*
|
||||||
|
* @param <E>
|
||||||
|
* @param <T>
|
||||||
|
* @param jsonObject
|
||||||
|
* @param object
|
||||||
|
* @param baseClass
|
||||||
|
* @param ignoreMissing
|
||||||
|
* @param members
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static <E extends Enum<?>, T> void fieldsFromJSON(
|
||||||
|
JSONObject jsonObject,
|
||||||
|
T object,
|
||||||
|
Class<? extends T> baseClass,
|
||||||
|
boolean ignoreMissing,
|
||||||
|
E... members)
|
||||||
|
throws JSONException {
|
||||||
|
try {
|
||||||
|
fieldsFromJSON(
|
||||||
|
jsonObject,
|
||||||
|
object,
|
||||||
|
baseClass,
|
||||||
|
ignoreMissing,
|
||||||
|
ClassUtil.getFieldsFromMembersEnum(baseClass, members));
|
||||||
|
} catch (NoSuchFieldException ex) {
|
||||||
|
throw new JSONException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For the given list of Fields, load in the values from the JSON object into the current object
|
||||||
|
* If ignore_missing is false, then JSONUtil will not throw an error if a field is missing
|
||||||
|
*
|
||||||
|
* @param <E>
|
||||||
|
* @param <T>
|
||||||
|
* @param jsonObject
|
||||||
|
* @param object
|
||||||
|
* @param baseClass
|
||||||
|
* @param ignoreMissing
|
||||||
|
* @param fields
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public static <E extends Enum<?>, T> void fieldsFromJSON(
|
||||||
|
JSONObject jsonObject,
|
||||||
|
T object,
|
||||||
|
Class<? extends T> baseClass,
|
||||||
|
boolean ignoreMissing,
|
||||||
|
Field... fields)
|
||||||
|
throws JSONException {
|
||||||
|
for (Field fieldHandle : fields) {
|
||||||
|
String jsonKey = fieldHandle.getName().toUpperCase();
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.debug("Retreiving value for field '" + jsonKey + "'");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!jsonObject.has(jsonKey)) {
|
||||||
|
String msg =
|
||||||
|
"JSONObject for "
|
||||||
|
+ baseClass.getSimpleName()
|
||||||
|
+ " does not have key '"
|
||||||
|
+ jsonKey
|
||||||
|
+ "': "
|
||||||
|
+ CollectionUtil.list(jsonObject.keys());
|
||||||
|
if (ignoreMissing) {
|
||||||
|
if (LOG.isDebugEnabled()) {
|
||||||
|
LOG.warn(msg);
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
} else {
|
||||||
|
throw new JSONException(msg);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
readFieldValue(jsonObject, jsonKey, fieldHandle, object);
|
||||||
|
} catch (Exception ex) {
|
||||||
|
// System.err.println(field_class + ": " + ClassUtil.getSuperClasses(field_class));
|
||||||
|
LOG.error(
|
||||||
|
"Unable to deserialize field '" + jsonKey + "' from " + baseClass.getSimpleName(),
|
||||||
|
ex);
|
||||||
|
throw new JSONException(ex);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the class of a field if it was stored in the JSONObject along with the value If there is
|
||||||
|
* no class information, then this will return null
|
||||||
|
*
|
||||||
|
* @param jsonObject
|
||||||
|
* @param jsonKey
|
||||||
|
* @return
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
private static Class<?> getClassForField(JSONObject jsonObject, String jsonKey)
|
||||||
|
throws JSONException {
|
||||||
|
Class<?> fieldClass = null;
|
||||||
|
// Check whether we also stored the class
|
||||||
|
if (jsonObject.has(jsonKey + JSON_CLASS_SUFFIX)) {
|
||||||
|
try {
|
||||||
|
fieldClass = ClassUtil.getClass(jsonObject.getString(jsonKey + JSON_CLASS_SUFFIX));
|
||||||
|
} catch (Exception ex) {
|
||||||
|
LOG.error("Failed to include class for field '" + jsonKey + "'", ex);
|
||||||
|
throw new JSONException(ex);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return (fieldClass);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the proper serialization string for the given value
|
||||||
|
*
|
||||||
|
* @param fieldClass
|
||||||
|
* @param fieldValue
|
||||||
|
* @return
|
||||||
|
*/
|
||||||
|
private static Object makePrimitiveValue(Class<?> fieldClass, Object fieldValue) {
|
||||||
|
Object value = null;
|
||||||
|
|
||||||
|
// Class
|
||||||
|
if (fieldClass.equals(Class.class)) {
|
||||||
|
value = ((Class<?>) fieldValue).getName();
|
||||||
|
// JSONSerializable
|
||||||
|
} else if (ClassUtil.getInterfaces(fieldClass).contains(JSONSerializable.class)) {
|
||||||
|
// Just return the value back. The JSON library will take care of it
|
||||||
|
// System.err.println(field_class + ": " + field_value);
|
||||||
|
value = fieldValue;
|
||||||
|
// Everything else
|
||||||
|
} else {
|
||||||
|
value = fieldValue; // .toString();
|
||||||
|
}
|
||||||
|
return (value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* For the given JSON string, figure out what kind of object it is and return it
|
||||||
|
*
|
||||||
|
* @param jsonValue
|
||||||
|
* @param fieldClass
|
||||||
|
* @return
|
||||||
|
* @throws Exception
|
||||||
|
*/
|
||||||
|
public static Object getPrimitiveValue(String jsonValue, Class<?> fieldClass) throws Exception {
|
||||||
|
Object value = null;
|
||||||
|
|
||||||
|
// Class
|
||||||
|
if (fieldClass.equals(Class.class)) {
|
||||||
|
value = ClassUtil.getClass(jsonValue);
|
||||||
|
if (value == null) {
|
||||||
|
throw new JSONException("Failed to get class from '" + jsonValue + "'");
|
||||||
|
}
|
||||||
|
// Enum
|
||||||
|
} else if (fieldClass.isEnum()) {
|
||||||
|
for (Object o : fieldClass.getEnumConstants()) {
|
||||||
|
Enum<?> e = (Enum<?>) o;
|
||||||
|
if (jsonValue.equals(e.name())) {
|
||||||
|
return (e);
|
||||||
|
}
|
||||||
|
} // FOR
|
||||||
|
throw new JSONException(
|
||||||
|
"Invalid enum value '"
|
||||||
|
+ jsonValue
|
||||||
|
+ "': "
|
||||||
|
+ Arrays.toString(fieldClass.getEnumConstants()));
|
||||||
|
// JSONSerializable
|
||||||
|
} else if (ClassUtil.getInterfaces(fieldClass).contains(JSONSerializable.class)) {
|
||||||
|
value = ClassUtil.newInstance(fieldClass, null, null);
|
||||||
|
((JSONSerializable) value).fromJSON(new JSONObject(jsonValue));
|
||||||
|
// Boolean
|
||||||
|
} else if (fieldClass.equals(Boolean.class) || fieldClass.equals(boolean.class)) {
|
||||||
|
// We have to use field_class.equals() because the value may be null
|
||||||
|
value = Boolean.parseBoolean(jsonValue);
|
||||||
|
// Short
|
||||||
|
} else if (fieldClass.equals(Short.class) || fieldClass.equals(short.class)) {
|
||||||
|
value = Short.parseShort(jsonValue);
|
||||||
|
// Integer
|
||||||
|
} else if (fieldClass.equals(Integer.class) || fieldClass.equals(int.class)) {
|
||||||
|
value = Integer.parseInt(jsonValue);
|
||||||
|
// Long
|
||||||
|
} else if (fieldClass.equals(Long.class) || fieldClass.equals(long.class)) {
|
||||||
|
value = Long.parseLong(jsonValue);
|
||||||
|
// Float
|
||||||
|
} else if (fieldClass.equals(Float.class) || fieldClass.equals(float.class)) {
|
||||||
|
value = Float.parseFloat(jsonValue);
|
||||||
|
// Double
|
||||||
|
} else if (fieldClass.equals(Double.class) || fieldClass.equals(double.class)) {
|
||||||
|
value = Double.parseDouble(jsonValue);
|
||||||
|
// String
|
||||||
|
} else if (fieldClass.equals(String.class)) {
|
||||||
|
value = jsonValue.toString();
|
||||||
|
}
|
||||||
|
return (value);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Class<?> getPrimitiveType(String jsonValue) {
|
||||||
|
Object value = null;
|
||||||
|
|
||||||
|
// CHECKSTYLE:OFF
|
||||||
|
// Class
|
||||||
|
try {
|
||||||
|
value = ClassUtil.getClass(jsonValue);
|
||||||
|
if (value != null) return (Class.class);
|
||||||
|
} catch (Throwable ex) {
|
||||||
|
} // IGNORE
|
||||||
|
|
||||||
|
// Short
|
||||||
|
try {
|
||||||
|
value = Short.parseShort(jsonValue);
|
||||||
|
return (Short.class);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
} // IGNORE
|
||||||
|
|
||||||
|
// Integer
|
||||||
|
try {
|
||||||
|
value = Integer.parseInt(jsonValue);
|
||||||
|
return (Integer.class);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
} // IGNORE
|
||||||
|
|
||||||
|
// Long
|
||||||
|
try {
|
||||||
|
value = Long.parseLong(jsonValue);
|
||||||
|
return (Long.class);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
} // IGNORE
|
||||||
|
|
||||||
|
// Float
|
||||||
|
try {
|
||||||
|
value = Float.parseFloat(jsonValue);
|
||||||
|
return (Float.class);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
} // IGNORE
|
||||||
|
|
||||||
|
// Double
|
||||||
|
try {
|
||||||
|
value = Double.parseDouble(jsonValue);
|
||||||
|
return (Double.class);
|
||||||
|
} catch (NumberFormatException ex) {
|
||||||
|
} // IGNORE
|
||||||
|
// CHECKSTYLE:ON
|
||||||
|
|
||||||
|
// Boolean
|
||||||
|
if (jsonValue.equalsIgnoreCase("true") || jsonValue.equalsIgnoreCase("false")) {
|
||||||
|
return (Boolean.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Default: String
|
||||||
|
return (String.class);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,138 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - ValidationUtils.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.JsonNode;
|
||||||
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||||
|
import com.github.fge.jackson.JsonLoader;
|
||||||
|
import com.github.fge.jsonschema.core.exceptions.ProcessingException;
|
||||||
|
import com.github.fge.jsonschema.core.report.ProcessingMessage;
|
||||||
|
import com.github.fge.jsonschema.core.report.ProcessingReport;
|
||||||
|
import com.github.fge.jsonschema.main.JsonSchema;
|
||||||
|
import com.github.fge.jsonschema.main.JsonSchemaFactory;
|
||||||
|
import java.io.File;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.net.URL;
|
||||||
|
|
||||||
|
public class ValidationUtils {
|
||||||
|
|
||||||
|
public static final String JSON_V4_SCHEMA_IDENTIFIER = "http://json-schema.org/draft-04/schema#";
|
||||||
|
public static final String JSON_SCHEMA_IDENTIFIER_ELEMENT = "$schema";
|
||||||
|
|
||||||
|
public static JsonNode getJsonNode(String jsonText) throws IOException {
|
||||||
|
return JsonLoader.fromString(jsonText);
|
||||||
|
} // getJsonNode(text) ends
|
||||||
|
|
||||||
|
public static JsonNode getJsonNode(File jsonFile) throws IOException {
|
||||||
|
return JsonLoader.fromFile(jsonFile);
|
||||||
|
} // getJsonNode(File) ends
|
||||||
|
|
||||||
|
public static JsonNode getJsonNode(URL url) throws IOException {
|
||||||
|
return JsonLoader.fromURL(url);
|
||||||
|
} // getJsonNode(URL) ends
|
||||||
|
|
||||||
|
public static JsonNode getJsonNodeFromResource(String resource) throws IOException {
|
||||||
|
return JsonLoader.fromResource(resource);
|
||||||
|
} // getJsonNode(Resource) ends
|
||||||
|
|
||||||
|
public static JsonSchema getSchemaNode(String schemaText)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonNode schemaNode = getJsonNode(schemaText);
|
||||||
|
return getSchemaNodeHelper(schemaNode);
|
||||||
|
} // getSchemaNode(text) ends
|
||||||
|
|
||||||
|
public static JsonSchema getSchemaNode(File schemaFile) throws IOException, ProcessingException {
|
||||||
|
final JsonNode schemaNode = getJsonNode(schemaFile);
|
||||||
|
return getSchemaNodeHelper(schemaNode);
|
||||||
|
} // getSchemaNode(File) ends
|
||||||
|
|
||||||
|
public static JsonSchema getSchemaNode(URL schemaFile) throws IOException, ProcessingException {
|
||||||
|
final JsonNode schemaNode = getJsonNode(schemaFile);
|
||||||
|
return getSchemaNodeHelper(schemaNode);
|
||||||
|
} // getSchemaNode(URL) ends
|
||||||
|
|
||||||
|
public static JsonSchema getSchemaNodeFromResource(String resource)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonNode schemaNode = getJsonNodeFromResource(resource);
|
||||||
|
return getSchemaNodeHelper(schemaNode);
|
||||||
|
} // getSchemaNode() ends
|
||||||
|
|
||||||
|
public static void validateJson(String schemaText, String jsonText)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaText);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonText);
|
||||||
|
validateJson(schemaNode, jsonNode);
|
||||||
|
} // validateJson(text) ends
|
||||||
|
|
||||||
|
public static void validateJson(File schemaFile, File jsonFile)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaFile);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonFile);
|
||||||
|
validateJson(schemaNode, jsonNode);
|
||||||
|
} // validateJson(File) ends
|
||||||
|
|
||||||
|
public static void validateJson(JsonSchema jsonSchemaNode, JsonNode jsonNode)
|
||||||
|
throws ProcessingException {
|
||||||
|
ProcessingReport report = jsonSchemaNode.validate(jsonNode);
|
||||||
|
if (!report.isSuccess()) {
|
||||||
|
for (ProcessingMessage processingMessage : report) {
|
||||||
|
throw new ProcessingException(processingMessage);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} // validateJson(Node) ends
|
||||||
|
|
||||||
|
public static void validateJson(URL schemaDocument, URL jsonDocument)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaDocument);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonDocument);
|
||||||
|
validateJson(schemaNode, jsonNode);
|
||||||
|
} // validateJson(URL) ends
|
||||||
|
|
||||||
|
public static boolean isJsonValid(JsonSchema jsonSchemaNode, JsonNode jsonNode)
|
||||||
|
throws ProcessingException {
|
||||||
|
ProcessingReport report = jsonSchemaNode.validate(jsonNode);
|
||||||
|
return report.isSuccess();
|
||||||
|
} // validateJson(Node) ends
|
||||||
|
|
||||||
|
public static boolean isJsonValid(String schemaText, String jsonText)
|
||||||
|
throws ProcessingException, IOException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaText);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonText);
|
||||||
|
return isJsonValid(schemaNode, jsonNode);
|
||||||
|
} // validateJson(Node) ends
|
||||||
|
|
||||||
|
public static boolean isJsonValid(File schemaFile, File jsonFile)
|
||||||
|
throws ProcessingException, IOException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaFile);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonFile);
|
||||||
|
return isJsonValid(schemaNode, jsonNode);
|
||||||
|
} // validateJson(Node) ends
|
||||||
|
|
||||||
|
public static boolean isJsonValid(URL schemaURL, URL jsonURL)
|
||||||
|
throws ProcessingException, IOException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaURL);
|
||||||
|
final JsonNode jsonNode = getJsonNode(jsonURL);
|
||||||
|
return isJsonValid(schemaNode, jsonNode);
|
||||||
|
} // validateJson(Node) ends
|
||||||
|
|
||||||
|
public static void validateJsonResource(String schemaResource, String jsonResource)
|
||||||
|
throws IOException, ProcessingException {
|
||||||
|
final JsonSchema schemaNode = getSchemaNode(schemaResource);
|
||||||
|
final JsonNode jsonNode = getJsonNodeFromResource(jsonResource);
|
||||||
|
validateJson(schemaNode, jsonNode);
|
||||||
|
} // validateJsonResource() ends
|
||||||
|
|
||||||
|
private static JsonSchema getSchemaNodeHelper(JsonNode jsonNode) throws ProcessingException {
|
||||||
|
final JsonNode schemaIdentifier = jsonNode.get(JSON_SCHEMA_IDENTIFIER_ELEMENT);
|
||||||
|
if (null == schemaIdentifier) {
|
||||||
|
((ObjectNode) jsonNode).put(JSON_SCHEMA_IDENTIFIER_ELEMENT, JSON_V4_SCHEMA_IDENTIFIER);
|
||||||
|
}
|
||||||
|
|
||||||
|
final JsonSchemaFactory factory = JsonSchemaFactory.byDefault();
|
||||||
|
return factory.getJsonSchema(jsonNode);
|
||||||
|
} // getSchemaNodeHelper() ends
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,862 @@
|
||||||
|
/*
|
||||||
|
Copyright (c) 2002 JSON.org
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
The Software shall be used for Good, not Evil.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Writer;
|
||||||
|
import java.lang.reflect.Array;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A JSONArray is an ordered sequence of values. Its external text form is a string wrapped in
|
||||||
|
* square brackets with commas separating the values. The internal form is an object having <code>
|
||||||
|
* get</code> and <code>opt</code> methods for accessing the values by index, and <code>put</code>
|
||||||
|
* methods for adding or replacing values. The values can be any of these types: <code>Boolean
|
||||||
|
* </code>, <code>JSONArray</code>, <code>JSONObject</code>, <code>Number</code>, <code>String
|
||||||
|
* </code>, or the <code>JSONObject.NULL object</code>.
|
||||||
|
*
|
||||||
|
* <p>The constructor can convert a JSON text into a Java object. The <code>toString</code> method
|
||||||
|
* converts to JSON text.
|
||||||
|
*
|
||||||
|
* <p>A <code>get</code> method returns a value if one can be found, and throws an exception if one
|
||||||
|
* cannot be found. An <code>opt</code> method returns a default value instead of throwing an
|
||||||
|
* exception, and so is useful for obtaining optional values.
|
||||||
|
*
|
||||||
|
* <p>The generic <code>get()</code> and <code>opt()</code> methods return an object which you can
|
||||||
|
* cast or query for type. There are also typed <code>get</code> and <code>opt</code> methods that
|
||||||
|
* do type checking and type coersion for you.
|
||||||
|
*
|
||||||
|
* <p>The texts produced by the <code>toString</code> methods strictly conform to JSON syntax rules.
|
||||||
|
* The constructors are more forgiving in the texts they will accept:
|
||||||
|
*
|
||||||
|
* <ul>
|
||||||
|
* <li>An extra <code>,</code> <small>(comma)</small> may appear just before the closing
|
||||||
|
* bracket.
|
||||||
|
* <li>The <code>null</code> value will be inserted when there is <code>,</code>
|
||||||
|
* <small>(comma)</small> elision.
|
||||||
|
* <li>Strings may be quoted with <code>'</code> <small>(single quote)</small>.
|
||||||
|
* <li>Strings do not need to be quoted at all if they do not begin with a quote or single quote,
|
||||||
|
* and if they do not contain leading or trailing spaces, and if they do not contain any of
|
||||||
|
* these characters: <code>{ } [ ] / \ : , = ; #</code> and if they do not look like numbers
|
||||||
|
* and if they are not the reserved words <code>true</code>, <code>false</code>, or <code>null
|
||||||
|
* </code>.
|
||||||
|
* <li>Values can be separated by <code>;</code> <small>(semicolon)</small> as well as by <code>,
|
||||||
|
* </code> <small>(comma)</small>.
|
||||||
|
* <li>Numbers may have the <code>0-</code> <small>(octal)</small> or <code>0x-</code>
|
||||||
|
* <small>(hex)</small> prefix.
|
||||||
|
* <li>Comments written in the slashshlash, slashstar, and hash conventions will be ignored.
|
||||||
|
* </ul>
|
||||||
|
*
|
||||||
|
* @author JSON.org
|
||||||
|
* @version 2008-09-18
|
||||||
|
*/
|
||||||
|
public class JSONArray {
|
||||||
|
|
||||||
|
/** The arrayList where the JSONArray's properties are kept. */
|
||||||
|
private ArrayList<Object> myArrayList;
|
||||||
|
|
||||||
|
/** Construct an empty JSONArray. */
|
||||||
|
public JSONArray() {
|
||||||
|
this.myArrayList = new ArrayList<Object>();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONArray from a JSONTokener.
|
||||||
|
*
|
||||||
|
* @param x A JSONTokener
|
||||||
|
* @throws JSONException If there is a syntax error.
|
||||||
|
*/
|
||||||
|
public JSONArray(JSONTokener x) throws JSONException {
|
||||||
|
this();
|
||||||
|
char c = x.nextClean();
|
||||||
|
char q;
|
||||||
|
if (c == '[') {
|
||||||
|
q = ']';
|
||||||
|
} else if (c == '(') {
|
||||||
|
q = ')';
|
||||||
|
} else {
|
||||||
|
throw x.syntaxError("A JSONArray text must start with '['");
|
||||||
|
}
|
||||||
|
if (x.nextClean() == ']') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
x.back();
|
||||||
|
for (; ; ) {
|
||||||
|
if (x.nextClean() == ',') {
|
||||||
|
x.back();
|
||||||
|
this.myArrayList.add(null);
|
||||||
|
} else {
|
||||||
|
x.back();
|
||||||
|
this.myArrayList.add(x.nextValue());
|
||||||
|
}
|
||||||
|
c = x.nextClean();
|
||||||
|
switch (c) {
|
||||||
|
case ';':
|
||||||
|
case ',':
|
||||||
|
if (x.nextClean() == ']') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
x.back();
|
||||||
|
break;
|
||||||
|
case ']':
|
||||||
|
case ')':
|
||||||
|
if (q != c) {
|
||||||
|
throw x.syntaxError("Expected a '" + new Character(q) + "'");
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
default:
|
||||||
|
throw x.syntaxError("Expected a ',' or ']'");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONArray from a source JSON text.
|
||||||
|
*
|
||||||
|
* @param source A string that begins with <code>[</code> <small>(left bracket)</small> and
|
||||||
|
* ends with <code>]</code> <small>(right bracket)</small>.
|
||||||
|
* @throws JSONException If there is a syntax error.
|
||||||
|
*/
|
||||||
|
public JSONArray(String source) throws JSONException {
|
||||||
|
this(new JSONTokener(source));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONArray from a Collection.
|
||||||
|
*
|
||||||
|
* @param collection A Collection.
|
||||||
|
*/
|
||||||
|
public JSONArray(Collection<?> collection) {
|
||||||
|
this.myArrayList =
|
||||||
|
(collection == null) ? new ArrayList<Object>() : new ArrayList<Object>(collection);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Construct a JSONArray from a collection of beans. The collection should have Java Beans. */
|
||||||
|
public JSONArray(Collection<?> collection, boolean includeSuperClass) {
|
||||||
|
this.myArrayList = new ArrayList<Object>();
|
||||||
|
if (collection != null) {
|
||||||
|
for (Iterator<?> iter = collection.iterator(); iter.hasNext(); ) {
|
||||||
|
this.myArrayList.add(new JSONObject(iter.next(), includeSuperClass));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONArray from an array
|
||||||
|
*
|
||||||
|
* @throws JSONException If not an array.
|
||||||
|
*/
|
||||||
|
public JSONArray(Object array) throws JSONException {
|
||||||
|
this();
|
||||||
|
if (array.getClass().isArray()) {
|
||||||
|
int length = Array.getLength(array);
|
||||||
|
for (int i = 0; i < length; i += 1) {
|
||||||
|
this.put(Array.get(array, i));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new JSONException("JSONArray initial value should be a string or collection or array.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONArray from an array with a bean. The array should have Java Beans.
|
||||||
|
*
|
||||||
|
* @throws JSONException If not an array.
|
||||||
|
*/
|
||||||
|
public JSONArray(Object array, boolean includeSuperClass) throws JSONException {
|
||||||
|
this();
|
||||||
|
if (array.getClass().isArray()) {
|
||||||
|
int length = Array.getLength(array);
|
||||||
|
for (int i = 0; i < length; i += 1) {
|
||||||
|
this.put(new JSONObject(Array.get(array, i), includeSuperClass));
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
throw new JSONException("JSONArray initial value should be a string or collection or array.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the object value associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return An object value.
|
||||||
|
* @throws JSONException If there is no value for the index.
|
||||||
|
*/
|
||||||
|
public Object get(int index) throws JSONException {
|
||||||
|
Object o = opt(index);
|
||||||
|
if (o == null) {
|
||||||
|
throw new JSONException("JSONArray[" + index + "] not found.");
|
||||||
|
}
|
||||||
|
return o;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the boolean value associated with an index. The string values "true" and "false" are
|
||||||
|
* converted to boolean.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The truth.
|
||||||
|
* @throws JSONException If there is no value for the index or if the value is not convertable to
|
||||||
|
* boolean.
|
||||||
|
*/
|
||||||
|
public boolean getBoolean(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
if (o.equals(Boolean.FALSE)
|
||||||
|
|| (o instanceof String && ((String) o).equalsIgnoreCase("false"))) {
|
||||||
|
return false;
|
||||||
|
} else if (o.equals(Boolean.TRUE)
|
||||||
|
|| (o instanceof String && ((String) o).equalsIgnoreCase("true"))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
throw new JSONException("JSONArray[" + index + "] is not a Boolean.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the double value associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
|
||||||
|
*/
|
||||||
|
public double getDouble(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
try {
|
||||||
|
return o instanceof Number
|
||||||
|
? ((Number) o).doubleValue()
|
||||||
|
: Double.valueOf((String) o).doubleValue();
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new JSONException("JSONArray[" + index + "] is not a number.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the int value associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
|
||||||
|
* if the value cannot be converted to a number.
|
||||||
|
*/
|
||||||
|
public int getInt(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
return o instanceof Number ? ((Number) o).intValue() : (int) getDouble(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the JSONArray associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return A JSONArray value.
|
||||||
|
* @throws JSONException If there is no value for the index. or if the value is not a JSONArray
|
||||||
|
*/
|
||||||
|
public JSONArray getJSONArray(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
if (o instanceof JSONArray) {
|
||||||
|
return (JSONArray) o;
|
||||||
|
}
|
||||||
|
throw new JSONException("JSONArray[" + index + "] is not a JSONArray.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the JSONObject associated with an index.
|
||||||
|
*
|
||||||
|
* @param index subscript
|
||||||
|
* @return A JSONObject value.
|
||||||
|
* @throws JSONException If there is no value for the index or if the value is not a JSONObject
|
||||||
|
*/
|
||||||
|
public JSONObject getJSONObject(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
if (o instanceof JSONObject) {
|
||||||
|
return (JSONObject) o;
|
||||||
|
}
|
||||||
|
throw new JSONException("JSONArray[" + index + "] is not a JSONObject.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the long value associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
* @throws JSONException If the key is not found or if the value cannot be converted to a number.
|
||||||
|
*/
|
||||||
|
public long getLong(int index) throws JSONException {
|
||||||
|
Object o = get(index);
|
||||||
|
return o instanceof Number ? ((Number) o).longValue() : (long) getDouble(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the string associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return A string value.
|
||||||
|
* @throws JSONException If there is no value for the index.
|
||||||
|
*/
|
||||||
|
public String getString(int index) throws JSONException {
|
||||||
|
return get(index).toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the value is null.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return true if the value at the index is null, or if there is no value.
|
||||||
|
*/
|
||||||
|
public boolean isNull(int index) {
|
||||||
|
return JSONObject.NULL.equals(opt(index));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a string from the contents of this JSONArray. The <code>separator</code> string is
|
||||||
|
* inserted between each element. Warning: This method assumes that the data structure is
|
||||||
|
* acyclical.
|
||||||
|
*
|
||||||
|
* @param separator A string that will be inserted between the elements.
|
||||||
|
* @return a string.
|
||||||
|
* @throws JSONException If the array contains an invalid number.
|
||||||
|
*/
|
||||||
|
public String join(String separator) throws JSONException {
|
||||||
|
int len = length();
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
|
||||||
|
for (int i = 0; i < len; i += 1) {
|
||||||
|
if (i > 0) {
|
||||||
|
sb.append(separator);
|
||||||
|
}
|
||||||
|
sb.append(JSONObject.valueToString(this.myArrayList.get(i)));
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the number of elements in the JSONArray, included nulls.
|
||||||
|
*
|
||||||
|
* @return The length (or size).
|
||||||
|
*/
|
||||||
|
public int length() {
|
||||||
|
return this.myArrayList.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional object value associated with an index.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return An object value, or null if there is no object at that index.
|
||||||
|
*/
|
||||||
|
public Object opt(int index) {
|
||||||
|
return (index < 0 || index >= length()) ? null : this.myArrayList.get(index);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional boolean value associated with an index. It returns false if there is no value
|
||||||
|
* at that index, or if the value is not Boolean.TRUE or the String "true".
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The truth.
|
||||||
|
*/
|
||||||
|
public boolean optBoolean(int index) {
|
||||||
|
return optBoolean(index, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional boolean value associated with an index. It returns the defaultValue if there
|
||||||
|
* is no value at that index or if it is not a Boolean or the String "true" or "false" (case
|
||||||
|
* insensitive).
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @param defaultValue A boolean default.
|
||||||
|
* @return The truth.
|
||||||
|
*/
|
||||||
|
public boolean optBoolean(int index, boolean defaultValue) {
|
||||||
|
try {
|
||||||
|
return getBoolean(index);
|
||||||
|
} catch (Exception e) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional double value associated with an index. NaN is returned if there is no value
|
||||||
|
* for the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public double optDouble(int index) {
|
||||||
|
return optDouble(index, Double.NaN);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional double value associated with an index. The defaultValue is returned if there
|
||||||
|
* is no value for the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index subscript
|
||||||
|
* @param defaultValue The default value.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public double optDouble(int index, double defaultValue) {
|
||||||
|
try {
|
||||||
|
return getDouble(index);
|
||||||
|
} catch (Exception e) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional int value associated with an index. Zero is returned if there is no value for
|
||||||
|
* the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public int optInt(int index) {
|
||||||
|
return optInt(index, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional int value associated with an index. The defaultValue is returned if there is
|
||||||
|
* no value for the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @param defaultValue The default value.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public int optInt(int index, int defaultValue) {
|
||||||
|
try {
|
||||||
|
return getInt(index);
|
||||||
|
} catch (Exception e) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional JSONArray associated with an index.
|
||||||
|
*
|
||||||
|
* @param index subscript
|
||||||
|
* @return A JSONArray value, or null if the index has no value, or if the value is not a
|
||||||
|
* JSONArray.
|
||||||
|
*/
|
||||||
|
public JSONArray optJSONArray(int index) {
|
||||||
|
Object o = opt(index);
|
||||||
|
return o instanceof JSONArray ? (JSONArray) o : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional JSONObject associated with an index. Null is returned if the key is not found,
|
||||||
|
* or null if the index has no value, or if the value is not a JSONObject.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return A JSONObject value.
|
||||||
|
*/
|
||||||
|
public JSONObject optJSONObject(int index) {
|
||||||
|
Object o = opt(index);
|
||||||
|
return o instanceof JSONObject ? (JSONObject) o : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional long value associated with an index. Zero is returned if there is no value for
|
||||||
|
* the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public long optLong(int index) {
|
||||||
|
return optLong(index, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional long value associated with an index. The defaultValue is returned if there is
|
||||||
|
* no value for the index, or if the value is not a number and cannot be converted to a number.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @param defaultValue The default value.
|
||||||
|
* @return The value.
|
||||||
|
*/
|
||||||
|
public long optLong(int index, long defaultValue) {
|
||||||
|
try {
|
||||||
|
return getLong(index);
|
||||||
|
} catch (Exception e) {
|
||||||
|
return defaultValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional string value associated with an index. It returns an empty string if there is
|
||||||
|
* no value at that index. If the value is not a string and is not null, then it is coverted to a
|
||||||
|
* string.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @return A String value.
|
||||||
|
*/
|
||||||
|
public String optString(int index) {
|
||||||
|
return optString(index, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the optional string associated with an index. The defaultValue is returned if the key is
|
||||||
|
* not found.
|
||||||
|
*
|
||||||
|
* @param index The index must be between 0 and length() - 1.
|
||||||
|
* @param defaultValue The default value.
|
||||||
|
* @return A String value.
|
||||||
|
*/
|
||||||
|
public String optString(int index, String defaultValue) {
|
||||||
|
Object o = opt(index);
|
||||||
|
return o != null ? o.toString() : defaultValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a boolean value. This increases the array's length by one.
|
||||||
|
*
|
||||||
|
* @param value A boolean value.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(boolean value) {
|
||||||
|
put(value ? Boolean.TRUE : Boolean.FALSE);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a value in the JSONArray, where the value will be a JSONArray which is produced from a
|
||||||
|
* Collection.
|
||||||
|
*
|
||||||
|
* @param value A Collection value.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(Collection<?> value) {
|
||||||
|
put(new JSONArray(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a double value. This increases the array's length by one.
|
||||||
|
*
|
||||||
|
* @param value A double value.
|
||||||
|
* @throws JSONException if the value is not finite.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(double value) throws JSONException {
|
||||||
|
Double d = new Double(value);
|
||||||
|
JSONObject.testValidity(d);
|
||||||
|
put(d);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append an int value. This increases the array's length by one.
|
||||||
|
*
|
||||||
|
* @param value An int value.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int value) {
|
||||||
|
put(new Integer(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append an long value. This increases the array's length by one.
|
||||||
|
*
|
||||||
|
* @param value A long value.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(long value) {
|
||||||
|
put(new Long(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a value in the JSONArray, where the value will be a JSONObject which is produced from a
|
||||||
|
* Map.
|
||||||
|
*
|
||||||
|
* @param value A Map value.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(Map<?, ?> value) {
|
||||||
|
put(new JSONObject(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append an object value. This increases the array's length by one.
|
||||||
|
*
|
||||||
|
* @param value An object value. The value should be a Boolean, Double, Integer, JSONArray,
|
||||||
|
* JSONObject, Long, or String, or the JSONObject.NULL object.
|
||||||
|
* @return this.
|
||||||
|
*/
|
||||||
|
public JSONArray put(Object value) {
|
||||||
|
this.myArrayList.add(value);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put or replace a boolean value in the JSONArray. If the index is greater than the length of the
|
||||||
|
* JSONArray, then null elements will be added as necessary to pad it out.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value A boolean value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, boolean value) throws JSONException {
|
||||||
|
put(index, value ? Boolean.TRUE : Boolean.FALSE);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a value in the JSONArray, where the value will be a JSONArray which is produced from a
|
||||||
|
* Collection.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value A Collection value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative or if the value is not finite.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, Collection<?> value) throws JSONException {
|
||||||
|
put(index, new JSONArray(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put or replace a double value. If the index is greater than the length of the JSONArray, then
|
||||||
|
* null elements will be added as necessary to pad it out.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value A double value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative or if the value is not finite.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, double value) throws JSONException {
|
||||||
|
put(index, new Double(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put or replace an int value. If the index is greater than the length of the JSONArray, then
|
||||||
|
* null elements will be added as necessary to pad it out.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value An int value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, int value) throws JSONException {
|
||||||
|
put(index, new Integer(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put or replace a long value. If the index is greater than the length of the JSONArray, then
|
||||||
|
* null elements will be added as necessary to pad it out.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value A long value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, long value) throws JSONException {
|
||||||
|
put(index, new Long(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put a value in the JSONArray, where the value will be a JSONObject which is produced from a
|
||||||
|
* Map.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value The Map value.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative or if the the value is an invalid number.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, Map<?, ?> value) throws JSONException {
|
||||||
|
put(index, new JSONObject(value));
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Put or replace an object value in the JSONArray. If the index is greater than the length of the
|
||||||
|
* JSONArray, then null elements will be added as necessary to pad it out.
|
||||||
|
*
|
||||||
|
* @param index The subscript.
|
||||||
|
* @param value The value to put into the array. The value should be a Boolean, Double, Integer,
|
||||||
|
* JSONArray, JSONObject, Long, or String, or the JSONObject.NULL object.
|
||||||
|
* @return this.
|
||||||
|
* @throws JSONException If the index is negative or if the the value is an invalid number.
|
||||||
|
*/
|
||||||
|
public JSONArray put(int index, Object value) throws JSONException {
|
||||||
|
JSONObject.testValidity(value);
|
||||||
|
if (index < 0) {
|
||||||
|
throw new JSONException("JSONArray[" + index + "] not found.");
|
||||||
|
}
|
||||||
|
if (index < length()) {
|
||||||
|
this.myArrayList.set(index, value);
|
||||||
|
} else {
|
||||||
|
while (index != length()) {
|
||||||
|
put(JSONObject.NULL);
|
||||||
|
}
|
||||||
|
put(value);
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Produce a JSONObject by combining a JSONArray of names with the values of this JSONArray.
|
||||||
|
*
|
||||||
|
* @param names A JSONArray containing a list of key strings. These will be paired with the
|
||||||
|
* values.
|
||||||
|
* @return A JSONObject, or null if there are no names or if this JSONArray has no values.
|
||||||
|
* @throws JSONException If any of the names are null.
|
||||||
|
*/
|
||||||
|
public JSONObject toJSONObject(JSONArray names) throws JSONException {
|
||||||
|
if (names == null || names.length() == 0 || length() == 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
JSONObject jo = new JSONObject();
|
||||||
|
for (int i = 0; i < names.length(); i += 1) {
|
||||||
|
jo.put(names.getString(i), this.opt(i));
|
||||||
|
}
|
||||||
|
return jo;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a JSON text of this JSONArray. For compactness, no unnecessary whitespace is added. If it
|
||||||
|
* is not possible to produce a syntactically correct JSON text then null will be returned
|
||||||
|
* instead. This could occur if the array contains an invalid number.
|
||||||
|
*
|
||||||
|
* <p>Warning: This method assumes that the data structure is acyclical.
|
||||||
|
*
|
||||||
|
* @return a printable, displayable, transmittable representation of the array.
|
||||||
|
*/
|
||||||
|
public String toString() {
|
||||||
|
try {
|
||||||
|
return '[' + join(",") + ']';
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a prettyprinted JSON text of this JSONArray. Warning: This method assumes that the data
|
||||||
|
* structure is acyclical.
|
||||||
|
*
|
||||||
|
* @param indentFactor The number of spaces to add to each level of indentation.
|
||||||
|
* @return a printable, displayable, transmittable representation of the object, beginning with
|
||||||
|
* <code>[</code> <small>(left bracket)</small> and ending with <code>]</code>
|
||||||
|
* <small>(right bracket)</small>.
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public String toString(int indentFactor) throws JSONException {
|
||||||
|
return toString(indentFactor, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a prettyprinted JSON text of this JSONArray. Warning: This method assumes that the data
|
||||||
|
* structure is acyclical.
|
||||||
|
*
|
||||||
|
* @param indentFactor The number of spaces to add to each level of indentation.
|
||||||
|
* @param indent The indention of the top level.
|
||||||
|
* @return a printable, displayable, transmittable representation of the array.
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
String toString(int indentFactor, int indent) throws JSONException {
|
||||||
|
int len = length();
|
||||||
|
if (len == 0) {
|
||||||
|
return "[]";
|
||||||
|
}
|
||||||
|
int i;
|
||||||
|
StringBuffer sb = new StringBuffer("[");
|
||||||
|
if (len == 1) {
|
||||||
|
sb.append(JSONObject.valueToString(this.myArrayList.get(0), indentFactor, indent));
|
||||||
|
} else {
|
||||||
|
int newindent = indent + indentFactor;
|
||||||
|
// sb.append('\n');
|
||||||
|
|
||||||
|
boolean intType = false;
|
||||||
|
for (i = 0; i < len; i += 1) {
|
||||||
|
if (this.myArrayList.get(i).getClass() != Integer.class) {
|
||||||
|
if (i == 0) {
|
||||||
|
sb.append('\n');
|
||||||
|
}
|
||||||
|
if (i > 0) {
|
||||||
|
sb.append(",\n");
|
||||||
|
}
|
||||||
|
for (int j = 0; j < newindent; j += 1) {
|
||||||
|
sb.append(' ');
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
intType = true;
|
||||||
|
if (i > 0) sb.append(", ");
|
||||||
|
}
|
||||||
|
sb.append(JSONObject.valueToString(this.myArrayList.get(i), indentFactor, newindent));
|
||||||
|
}
|
||||||
|
if (intType == false) {
|
||||||
|
sb.append('\n');
|
||||||
|
for (i = 0; i < indent; i += 1) {
|
||||||
|
sb.append(' ');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sb.append(']');
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write the contents of the JSONArray as JSON text to a writer. For compactness, no whitespace is
|
||||||
|
* added.
|
||||||
|
*
|
||||||
|
* <p>Warning: This method assumes that the data structure is acyclical.
|
||||||
|
*
|
||||||
|
* @return The writer.
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public Writer write(Writer writer) throws JSONException {
|
||||||
|
try {
|
||||||
|
boolean b = false;
|
||||||
|
int len = length();
|
||||||
|
|
||||||
|
writer.write('[');
|
||||||
|
|
||||||
|
for (int i = 0; i < len; i += 1) {
|
||||||
|
if (b) {
|
||||||
|
writer.write(',');
|
||||||
|
}
|
||||||
|
Object v = this.myArrayList.get(i);
|
||||||
|
if (v instanceof JSONObject) {
|
||||||
|
((JSONObject) v).write(writer);
|
||||||
|
} else if (v instanceof JSONArray) {
|
||||||
|
((JSONArray) v).write(writer);
|
||||||
|
} else {
|
||||||
|
writer.write(JSONObject.valueToString(v));
|
||||||
|
}
|
||||||
|
b = true;
|
||||||
|
}
|
||||||
|
writer.write(']');
|
||||||
|
return writer;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new JSONException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,30 @@
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The JSONException is thrown by the JSON.org classes then things are amiss.
|
||||||
|
*
|
||||||
|
* @author JSON.org
|
||||||
|
* @version 2008-09-18
|
||||||
|
*/
|
||||||
|
public class JSONException extends Exception {
|
||||||
|
private static final long serialVersionUID = 1L;
|
||||||
|
private Throwable cause;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Constructs a JSONException with an explanatory message.
|
||||||
|
*
|
||||||
|
* @param message Detail about the reason for the exception.
|
||||||
|
*/
|
||||||
|
public JSONException(String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public JSONException(Throwable t) {
|
||||||
|
super(t.getMessage());
|
||||||
|
this.cause = t;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Throwable getCause() {
|
||||||
|
return this.cause;
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,16 @@
|
||||||
|
package com.controller.util.json;
|
||||||
|
/**
|
||||||
|
* The <code>JSONString</code> interface allows a <code>toJSONString()</code> method so that a class
|
||||||
|
* can change the behavior of <code>JSONObject.toString()</code>, <code>JSONArray.toString()</code>,
|
||||||
|
* and <code>JSONWriter.value(</code>Object<code>)</code>. The <code>toJSONString</code> method will
|
||||||
|
* be used instead of the default behavior of using the Object's <code>toString()</code> method and
|
||||||
|
* quoting the result.
|
||||||
|
*/
|
||||||
|
public interface JSONString {
|
||||||
|
/**
|
||||||
|
* The <code>toJSONString</code> method allows a class to produce its own JSON serialization.
|
||||||
|
*
|
||||||
|
* @return A strictly syntactically correct JSON text.
|
||||||
|
*/
|
||||||
|
public String toJSONString();
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
Copyright (c) 2006 JSON.org
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
The Software shall be used for Good, not Evil.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
import java.io.StringWriter;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSONStringer provides a quick and convenient way of producing JSON text. The texts produced
|
||||||
|
* strictly conform to JSON syntax rules. No whitespace is added, so the results are ready for
|
||||||
|
* transmission or storage. Each instance of JSONStringer can produce one JSON text.
|
||||||
|
*
|
||||||
|
* <p>A JSONStringer instance provides a <code>value</code> method for appending values to the text,
|
||||||
|
* and a <code>key</code> method for adding keys before values in objects. There are <code>array
|
||||||
|
* </code> and <code>endArray</code> methods that make and bound array values, and <code>object
|
||||||
|
* </code> and <code>endObject</code> methods which make and bound object values. All of these
|
||||||
|
* methods return the JSONWriter instance, permitting cascade style. For example,
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* myString = new JSONStringer()
|
||||||
|
* .object()
|
||||||
|
* .key("JSON")
|
||||||
|
* .value("Hello, World!")
|
||||||
|
* .endObject()
|
||||||
|
* .toString();</pre>
|
||||||
|
*
|
||||||
|
* which produces the string
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* {"JSON":"Hello, World!"}</pre>
|
||||||
|
*
|
||||||
|
* <p>The first method called must be <code>array</code> or <code>object</code>. There are no
|
||||||
|
* methods for adding commas or colons. JSONStringer adds them for you. Objects and arrays can be
|
||||||
|
* nested up to 20 levels deep.
|
||||||
|
*
|
||||||
|
* <p>This can sometimes be easier than using a JSONObject to build a string.
|
||||||
|
*
|
||||||
|
* @author JSON.org
|
||||||
|
* @version 2008-09-18
|
||||||
|
*/
|
||||||
|
public class JSONStringer extends JSONWriter {
|
||||||
|
/** Make a fresh JSONStringer. It can be used to build one JSON text. */
|
||||||
|
public JSONStringer() {
|
||||||
|
super(new StringWriter());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the JSON text. This method is used to obtain the product of the JSONStringer instance.
|
||||||
|
* It will return <code>null</code> if there was a problem in the construction of the JSON text
|
||||||
|
* (such as the calls to <code>array</code> were not properly balanced with calls to <code>
|
||||||
|
* endArray</code>).
|
||||||
|
*
|
||||||
|
* @return The JSON text.
|
||||||
|
*/
|
||||||
|
public String toString() {
|
||||||
|
return this.mode == 'd' ? this.writer.toString() : null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,403 @@
|
||||||
|
/*
|
||||||
|
Copyright (c) 2002 JSON.org
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
The Software shall be used for Good, not Evil.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Reader;
|
||||||
|
import java.io.StringReader;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A JSONTokener takes a source string and extracts characters and tokens from it. It is used by the
|
||||||
|
* JSONObject and JSONArray constructors to parse JSON source strings.
|
||||||
|
*
|
||||||
|
* @author JSON.org
|
||||||
|
* @version 2008-09-18
|
||||||
|
*/
|
||||||
|
public class JSONTokener {
|
||||||
|
|
||||||
|
private int index;
|
||||||
|
private Reader reader;
|
||||||
|
private char lastChar;
|
||||||
|
private boolean useLastChar;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONTokener from a string.
|
||||||
|
*
|
||||||
|
* @param reader A reader.
|
||||||
|
*/
|
||||||
|
public JSONTokener(Reader reader) {
|
||||||
|
this.reader = reader.markSupported() ? reader : new BufferedReader(reader);
|
||||||
|
this.useLastChar = false;
|
||||||
|
this.index = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Construct a JSONTokener from a string.
|
||||||
|
*
|
||||||
|
* @param s A source string.
|
||||||
|
*/
|
||||||
|
public JSONTokener(String s) {
|
||||||
|
this(new StringReader(s));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Back up one character. This provides a sort of lookahead capability, so that you can test for a
|
||||||
|
* digit or letter before attempting to parse the next number or identifier.
|
||||||
|
*/
|
||||||
|
public void back() throws JSONException {
|
||||||
|
if (useLastChar || index <= 0) {
|
||||||
|
throw new JSONException("Stepping back two steps is not supported");
|
||||||
|
}
|
||||||
|
index -= 1;
|
||||||
|
useLastChar = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the hex value of a character (base16).
|
||||||
|
*
|
||||||
|
* @param c A character between '0' and '9' or between 'A' and 'F' or between 'a' and 'f'.
|
||||||
|
* @return An int between 0 and 15, or -1 if c was not a hex digit.
|
||||||
|
*/
|
||||||
|
public static int dehexchar(char c) {
|
||||||
|
if (c >= '0' && c <= '9') {
|
||||||
|
return c - '0';
|
||||||
|
}
|
||||||
|
if (c >= 'A' && c <= 'F') {
|
||||||
|
return c - ('A' - 10);
|
||||||
|
}
|
||||||
|
if (c >= 'a' && c <= 'f') {
|
||||||
|
return c - ('a' - 10);
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determine if the source string still contains characters that next() can consume.
|
||||||
|
*
|
||||||
|
* @return true if not yet at the end of the source.
|
||||||
|
*/
|
||||||
|
public boolean more() throws JSONException {
|
||||||
|
char nextChar = next();
|
||||||
|
if (nextChar == 0) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
back();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the next character in the source string.
|
||||||
|
*
|
||||||
|
* @return The next character, or 0 if past the end of the source string.
|
||||||
|
*/
|
||||||
|
public char next() throws JSONException {
|
||||||
|
if (this.useLastChar) {
|
||||||
|
this.useLastChar = false;
|
||||||
|
if (this.lastChar != 0) {
|
||||||
|
this.index += 1;
|
||||||
|
}
|
||||||
|
return this.lastChar;
|
||||||
|
}
|
||||||
|
int c;
|
||||||
|
try {
|
||||||
|
c = this.reader.read();
|
||||||
|
} catch (IOException exc) {
|
||||||
|
throw new JSONException(exc);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (c <= 0) { // End of stream
|
||||||
|
this.lastChar = 0;
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
this.index += 1;
|
||||||
|
this.lastChar = (char) c;
|
||||||
|
return this.lastChar;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Consume the next character, and check that it matches a specified character.
|
||||||
|
*
|
||||||
|
* @param c The character to match.
|
||||||
|
* @return The character.
|
||||||
|
* @throws JSONException if the character does not match.
|
||||||
|
*/
|
||||||
|
public char next(char c) throws JSONException {
|
||||||
|
char n = next();
|
||||||
|
if (n != c) {
|
||||||
|
throw syntaxError("Expected '" + c + "' and instead saw '" + n + "'");
|
||||||
|
}
|
||||||
|
return n;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the next n characters.
|
||||||
|
*
|
||||||
|
* @param n The number of characters to take.
|
||||||
|
* @return A string of n characters.
|
||||||
|
* @throws JSONException Substring bounds error if there are not n characters remaining in the
|
||||||
|
* source string.
|
||||||
|
*/
|
||||||
|
public String next(int n) throws JSONException {
|
||||||
|
if (n == 0) {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
char[] buffer = new char[n];
|
||||||
|
int pos = 0;
|
||||||
|
|
||||||
|
if (this.useLastChar) {
|
||||||
|
this.useLastChar = false;
|
||||||
|
buffer[0] = this.lastChar;
|
||||||
|
pos = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
int len;
|
||||||
|
while ((pos < n) && ((len = reader.read(buffer, pos, n - pos)) != -1)) {
|
||||||
|
pos += len;
|
||||||
|
}
|
||||||
|
} catch (IOException exc) {
|
||||||
|
throw new JSONException(exc);
|
||||||
|
}
|
||||||
|
this.index += pos;
|
||||||
|
|
||||||
|
if (pos < n) {
|
||||||
|
throw syntaxError("Substring bounds error");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.lastChar = buffer[n - 1];
|
||||||
|
return new String(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the next char in the string, skipping whitespace.
|
||||||
|
*
|
||||||
|
* @throws JSONException
|
||||||
|
* @return A character, or 0 if there are no more characters.
|
||||||
|
*/
|
||||||
|
public char nextClean() throws JSONException {
|
||||||
|
for (; ; ) {
|
||||||
|
char c = next();
|
||||||
|
if (c == 0 || c > ' ') {
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Return the characters up to the next close quote character. Backslash processing is done. The
|
||||||
|
* formal JSON format does not allow strings in single quotes, but an implementation is allowed to
|
||||||
|
* accept them.
|
||||||
|
*
|
||||||
|
* @param quote The quoting character, either <code>"</code> <small>(double quote)</small> or
|
||||||
|
* <code>'</code> <small>(single quote)</small>.
|
||||||
|
* @return A String.
|
||||||
|
* @throws JSONException Unterminated string.
|
||||||
|
*/
|
||||||
|
public String nextString(char quote) throws JSONException {
|
||||||
|
char c;
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
for (; ; ) {
|
||||||
|
c = next();
|
||||||
|
switch (c) {
|
||||||
|
case 0:
|
||||||
|
case '\n':
|
||||||
|
case '\r':
|
||||||
|
throw syntaxError("Unterminated string");
|
||||||
|
case '\\':
|
||||||
|
c = next();
|
||||||
|
switch (c) {
|
||||||
|
case 'b':
|
||||||
|
sb.append('\b');
|
||||||
|
break;
|
||||||
|
case 't':
|
||||||
|
sb.append('\t');
|
||||||
|
break;
|
||||||
|
case 'n':
|
||||||
|
sb.append('\n');
|
||||||
|
break;
|
||||||
|
case 'f':
|
||||||
|
sb.append('\f');
|
||||||
|
break;
|
||||||
|
case 'r':
|
||||||
|
sb.append('\r');
|
||||||
|
break;
|
||||||
|
case 'u':
|
||||||
|
sb.append((char) Integer.parseInt(next(4), 16));
|
||||||
|
break;
|
||||||
|
case 'x':
|
||||||
|
sb.append((char) Integer.parseInt(next(2), 16));
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
sb.append(c);
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
default:
|
||||||
|
if (c == quote) {
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
sb.append(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the text up but not including the specified character or the end of line, whichever comes
|
||||||
|
* first.
|
||||||
|
*
|
||||||
|
* @param d A delimiter character.
|
||||||
|
* @return A string.
|
||||||
|
*/
|
||||||
|
public String nextTo(char d) throws JSONException {
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
for (; ; ) {
|
||||||
|
char c = next();
|
||||||
|
if (c == d || c == 0 || c == '\n' || c == '\r') {
|
||||||
|
if (c != 0) {
|
||||||
|
back();
|
||||||
|
}
|
||||||
|
return sb.toString().trim();
|
||||||
|
}
|
||||||
|
sb.append(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the text up but not including one of the specified delimiter characters or the end of line,
|
||||||
|
* whichever comes first.
|
||||||
|
*
|
||||||
|
* @param delimiters A set of delimiter characters.
|
||||||
|
* @return A string, trimmed.
|
||||||
|
*/
|
||||||
|
public String nextTo(String delimiters) throws JSONException {
|
||||||
|
char c;
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
for (; ; ) {
|
||||||
|
c = next();
|
||||||
|
if (delimiters.indexOf(c) >= 0 || c == 0 || c == '\n' || c == '\r') {
|
||||||
|
if (c != 0) {
|
||||||
|
back();
|
||||||
|
}
|
||||||
|
return sb.toString().trim();
|
||||||
|
}
|
||||||
|
sb.append(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the next value. The value can be a Boolean, Double, Integer, JSONArray, JSONObject, Long,
|
||||||
|
* or String, or the JSONObject.NULL object.
|
||||||
|
*
|
||||||
|
* @throws JSONException If syntax error.
|
||||||
|
* @return An object.
|
||||||
|
*/
|
||||||
|
public Object nextValue() throws JSONException {
|
||||||
|
char c = nextClean();
|
||||||
|
String s;
|
||||||
|
|
||||||
|
switch (c) {
|
||||||
|
case '"':
|
||||||
|
case '\'':
|
||||||
|
return nextString(c);
|
||||||
|
case '{':
|
||||||
|
back();
|
||||||
|
return new JSONObject(this);
|
||||||
|
case '[':
|
||||||
|
case '(':
|
||||||
|
back();
|
||||||
|
return new JSONArray(this);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Handle unquoted text. This could be the values true, false, or
|
||||||
|
* null, or it can be a number. An implementation (such as this one)
|
||||||
|
* is allowed to also accept non-standard forms.
|
||||||
|
*
|
||||||
|
* Accumulate characters until we reach the end of the text or a
|
||||||
|
* formatting character.
|
||||||
|
*/
|
||||||
|
|
||||||
|
StringBuffer sb = new StringBuffer();
|
||||||
|
while (c >= ' ' && ",:]}/\\\"[{;=#".indexOf(c) < 0) {
|
||||||
|
sb.append(c);
|
||||||
|
c = next();
|
||||||
|
}
|
||||||
|
back();
|
||||||
|
|
||||||
|
s = sb.toString().trim();
|
||||||
|
if (s.equals("")) {
|
||||||
|
throw syntaxError("Missing value");
|
||||||
|
}
|
||||||
|
return JSONObject.stringToValue(s);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Skip characters until the next character is the requested character. If the requested character
|
||||||
|
* is not found, no characters are skipped.
|
||||||
|
*
|
||||||
|
* @param to A character to skip to.
|
||||||
|
* @return The requested character, or zero if the requested character is not found.
|
||||||
|
*/
|
||||||
|
public char skipTo(char to) throws JSONException {
|
||||||
|
char c;
|
||||||
|
try {
|
||||||
|
int startIndex = this.index;
|
||||||
|
reader.mark(Integer.MAX_VALUE);
|
||||||
|
do {
|
||||||
|
c = next();
|
||||||
|
if (c == 0) {
|
||||||
|
reader.reset();
|
||||||
|
this.index = startIndex;
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
} while (c != to);
|
||||||
|
} catch (IOException exc) {
|
||||||
|
throw new JSONException(exc);
|
||||||
|
}
|
||||||
|
|
||||||
|
back();
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a JSONException to signal a syntax error.
|
||||||
|
*
|
||||||
|
* @param message The error message.
|
||||||
|
* @return A JSONException object, suitable for throwing
|
||||||
|
*/
|
||||||
|
public JSONException syntaxError(String message) {
|
||||||
|
return new JSONException(message + toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make a printable string of this JSONTokener.
|
||||||
|
*
|
||||||
|
* @return " at character [this.index]"
|
||||||
|
*/
|
||||||
|
public String toString() {
|
||||||
|
return " at character " + index;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,309 @@
|
||||||
|
/*
|
||||||
|
Copyright (c) 2006 JSON.org
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
The Software shall be used for Good, not Evil.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.Writer;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* JSONWriter provides a quick and convenient way of producing JSON text. The texts produced
|
||||||
|
* strictly conform to JSON syntax rules. No whitespace is added, so the results are ready for
|
||||||
|
* transmission or storage. Each instance of JSONWriter can produce one JSON text.
|
||||||
|
*
|
||||||
|
* <p>A JSONWriter instance provides a <code>value</code> method for appending values to the text,
|
||||||
|
* and a <code>key</code> method for adding keys before values in objects. There are <code>array
|
||||||
|
* </code> and <code>endArray</code> methods that make and bound array values, and <code>object
|
||||||
|
* </code> and <code>endObject</code> methods which make and bound object values. All of these
|
||||||
|
* methods return the JSONWriter instance, permitting a cascade style. For example,
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* new JSONWriter(myWriter)
|
||||||
|
* .object()
|
||||||
|
* .key("JSON")
|
||||||
|
* .value("Hello, World!")
|
||||||
|
* .endObject();</pre>
|
||||||
|
*
|
||||||
|
* which writes
|
||||||
|
*
|
||||||
|
* <pre>
|
||||||
|
* {"JSON":"Hello, World!"}</pre>
|
||||||
|
*
|
||||||
|
* <p>The first method called must be <code>array</code> or <code>object</code>. There are no
|
||||||
|
* methods for adding commas or colons. JSONWriter adds them for you. Objects and arrays can be
|
||||||
|
* nested up to 20 levels deep.
|
||||||
|
*
|
||||||
|
* <p>This can sometimes be easier than using a JSONObject to build a string.
|
||||||
|
*
|
||||||
|
* @author JSON.org
|
||||||
|
* @version 2008-09-18
|
||||||
|
*/
|
||||||
|
public class JSONWriter {
|
||||||
|
private static final int maxdepth = 20;
|
||||||
|
|
||||||
|
/** The comma flag determines if a comma should be output before the next value. */
|
||||||
|
private boolean comma;
|
||||||
|
|
||||||
|
/** The current mode. Values: 'a' (array), 'd' (done), 'i' (initial), 'k' (key), 'o' (object). */
|
||||||
|
protected char mode;
|
||||||
|
|
||||||
|
/** The object/array stack. */
|
||||||
|
private JSONObject stack[];
|
||||||
|
|
||||||
|
/** The stack top index. A value of 0 indicates that the stack is empty. */
|
||||||
|
private int top;
|
||||||
|
|
||||||
|
/** The writer that will receive the output. */
|
||||||
|
protected Writer writer;
|
||||||
|
|
||||||
|
/** Make a fresh JSONWriter. It can be used to build one JSON text. */
|
||||||
|
public JSONWriter(Writer w) {
|
||||||
|
this.comma = false;
|
||||||
|
this.mode = 'i';
|
||||||
|
this.stack = new JSONObject[maxdepth];
|
||||||
|
this.top = 0;
|
||||||
|
this.writer = w;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a value.
|
||||||
|
*
|
||||||
|
* @param s A string value.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the value is out of sequence.
|
||||||
|
*/
|
||||||
|
private JSONWriter append(String s) throws JSONException {
|
||||||
|
if (s == null) {
|
||||||
|
throw new JSONException("Null pointer");
|
||||||
|
}
|
||||||
|
if (this.mode == 'o' || this.mode == 'a') {
|
||||||
|
try {
|
||||||
|
if (this.comma && this.mode == 'a') {
|
||||||
|
this.writer.write(',');
|
||||||
|
}
|
||||||
|
this.writer.write(s);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new JSONException(e);
|
||||||
|
}
|
||||||
|
if (this.mode == 'o') {
|
||||||
|
this.mode = 'k';
|
||||||
|
}
|
||||||
|
this.comma = true;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
throw new JSONException("Value out of sequence.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Begin appending a new array. All values until the balancing <code>endArray</code> will be
|
||||||
|
* appended to this array. The <code>endArray</code> method must be called to mark the array's
|
||||||
|
* end.
|
||||||
|
*
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the nesting is too deep, or if the object is started in the wrong
|
||||||
|
* place (for example as a key or after the end of the outermost array or object).
|
||||||
|
*/
|
||||||
|
public JSONWriter array() throws JSONException {
|
||||||
|
if (this.mode == 'i' || this.mode == 'o' || this.mode == 'a') {
|
||||||
|
this.push(null);
|
||||||
|
this.append("[");
|
||||||
|
this.comma = false;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
throw new JSONException("Misplaced array.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* End something.
|
||||||
|
*
|
||||||
|
* @param m Mode
|
||||||
|
* @param c Closing character
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If unbalanced.
|
||||||
|
*/
|
||||||
|
private JSONWriter end(char m, char c) throws JSONException {
|
||||||
|
if (this.mode != m) {
|
||||||
|
throw new JSONException(m == 'o' ? "Misplaced endObject." : "Misplaced endArray.");
|
||||||
|
}
|
||||||
|
this.pop(m);
|
||||||
|
try {
|
||||||
|
this.writer.write(c);
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new JSONException(e);
|
||||||
|
}
|
||||||
|
this.comma = true;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* End an array. This method most be called to balance calls to <code>array</code>.
|
||||||
|
*
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If incorrectly nested.
|
||||||
|
*/
|
||||||
|
public JSONWriter endArray() throws JSONException {
|
||||||
|
return this.end('a', ']');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* End an object. This method most be called to balance calls to <code>object</code>.
|
||||||
|
*
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If incorrectly nested.
|
||||||
|
*/
|
||||||
|
public JSONWriter endObject() throws JSONException {
|
||||||
|
return this.end('k', '}');
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a key. The key will be associated with the next value. In an object, every value must be
|
||||||
|
* preceded by a key.
|
||||||
|
*
|
||||||
|
* @param s A key string.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the key is out of place. For example, keys do not belong in arrays or
|
||||||
|
* if the key is null.
|
||||||
|
*/
|
||||||
|
public JSONWriter key(String s) throws JSONException {
|
||||||
|
if (s == null) {
|
||||||
|
throw new JSONException("Null key.");
|
||||||
|
}
|
||||||
|
if (this.mode == 'k') {
|
||||||
|
try {
|
||||||
|
if (this.comma) {
|
||||||
|
this.writer.write(',');
|
||||||
|
}
|
||||||
|
stack[top - 1].putOnce(s, Boolean.TRUE);
|
||||||
|
this.writer.write(JSONObject.quote(s));
|
||||||
|
this.writer.write(':');
|
||||||
|
this.comma = false;
|
||||||
|
this.mode = 'o';
|
||||||
|
return this;
|
||||||
|
} catch (IOException e) {
|
||||||
|
throw new JSONException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new JSONException("Misplaced key.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Begin appending a new object. All keys and values until the balancing <code>endObject</code>
|
||||||
|
* will be appended to this object. The <code>endObject</code> method must be called to mark the
|
||||||
|
* object's end.
|
||||||
|
*
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the nesting is too deep, or if the object is started in the wrong
|
||||||
|
* place (for example as a key or after the end of the outermost array or object).
|
||||||
|
*/
|
||||||
|
public JSONWriter object() throws JSONException {
|
||||||
|
if (this.mode == 'i') {
|
||||||
|
this.mode = 'o';
|
||||||
|
}
|
||||||
|
if (this.mode == 'o' || this.mode == 'a') {
|
||||||
|
this.append("{");
|
||||||
|
this.push(new JSONObject());
|
||||||
|
this.comma = false;
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
throw new JSONException("Misplaced object.");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Pop an array or object scope.
|
||||||
|
*
|
||||||
|
* @param c The scope to close.
|
||||||
|
* @throws JSONException If nesting is wrong.
|
||||||
|
*/
|
||||||
|
private void pop(char c) throws JSONException {
|
||||||
|
if (this.top <= 0) {
|
||||||
|
throw new JSONException("Nesting error.");
|
||||||
|
}
|
||||||
|
char m = this.stack[this.top - 1] == null ? 'a' : 'k';
|
||||||
|
if (m != c) {
|
||||||
|
throw new JSONException("Nesting error.");
|
||||||
|
}
|
||||||
|
this.top -= 1;
|
||||||
|
this.mode = this.top == 0 ? 'd' : this.stack[this.top - 1] == null ? 'a' : 'k';
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Push an array or object scope.
|
||||||
|
*
|
||||||
|
* @param c The scope to open.
|
||||||
|
* @throws JSONException If nesting is too deep.
|
||||||
|
*/
|
||||||
|
private void push(JSONObject jo) throws JSONException {
|
||||||
|
if (this.top >= maxdepth) {
|
||||||
|
throw new JSONException("Nesting too deep.");
|
||||||
|
}
|
||||||
|
this.stack[this.top] = jo;
|
||||||
|
this.mode = jo == null ? 'a' : 'k';
|
||||||
|
this.top += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append either the value <code>true</code> or the value <code>false</code>.
|
||||||
|
*
|
||||||
|
* @param b A boolean.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public JSONWriter value(boolean b) throws JSONException {
|
||||||
|
return this.append(b ? "true" : "false");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a double value.
|
||||||
|
*
|
||||||
|
* @param d A double.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the number is not finite.
|
||||||
|
*/
|
||||||
|
public JSONWriter value(double d) throws JSONException {
|
||||||
|
return this.value(new Double(d));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append a long value.
|
||||||
|
*
|
||||||
|
* @param l A long.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException
|
||||||
|
*/
|
||||||
|
public JSONWriter value(long l) throws JSONException {
|
||||||
|
return this.append(Long.toString(l));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append an object value.
|
||||||
|
*
|
||||||
|
* @param o The object to append. It can be null, or a Boolean, Number, String, JSONObject, or
|
||||||
|
* JSONArray, or an object with a toJSONString() method.
|
||||||
|
* @return this
|
||||||
|
* @throws JSONException If the value is out of sequence.
|
||||||
|
*/
|
||||||
|
public JSONWriter value(Object o) throws JSONException {
|
||||||
|
return this.append(JSONObject.valueToString(o));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,440 @@
|
||||||
|
package com.controller.util.json;
|
||||||
|
|
||||||
|
import java.io.StringWriter;
|
||||||
|
import java.util.Collection;
|
||||||
|
import java.util.Iterator;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Test class. This file is not formally a member of the com.controller.util.json library. It is
|
||||||
|
* just a casual test tool.
|
||||||
|
*/
|
||||||
|
public class Test {
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Entry point.
|
||||||
|
*
|
||||||
|
* @param args
|
||||||
|
*/
|
||||||
|
public static void main(String args[]) {
|
||||||
|
Iterator<?> it;
|
||||||
|
JSONArray a;
|
||||||
|
JSONObject j;
|
||||||
|
JSONStringer jj;
|
||||||
|
String s;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Obj is a typical class that implements JSONString. It also provides some beanie methods that
|
||||||
|
* can be used to construct a JSONObject. It also demonstrates constructing a JSONObject with an
|
||||||
|
* array of names.
|
||||||
|
*/
|
||||||
|
class Obj implements JSONString {
|
||||||
|
public String aString;
|
||||||
|
public double aNumber;
|
||||||
|
public boolean aBoolean;
|
||||||
|
|
||||||
|
public Obj(String string, double n, boolean b) {
|
||||||
|
this.aString = string;
|
||||||
|
this.aNumber = n;
|
||||||
|
this.aBoolean = b;
|
||||||
|
}
|
||||||
|
|
||||||
|
public double getNumber() {
|
||||||
|
return this.aNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getString() {
|
||||||
|
return this.aString;
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean isBoolean() {
|
||||||
|
return this.aBoolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getBENT() {
|
||||||
|
return "All uppercase key";
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getX() {
|
||||||
|
return "x";
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toJSONString() {
|
||||||
|
return "{"
|
||||||
|
+ JSONObject.quote(this.aString)
|
||||||
|
+ ":"
|
||||||
|
+ JSONObject.doubleToString(this.aNumber)
|
||||||
|
+ "}";
|
||||||
|
}
|
||||||
|
|
||||||
|
public String toString() {
|
||||||
|
return this.getString()
|
||||||
|
+ " "
|
||||||
|
+ this.getNumber()
|
||||||
|
+ " "
|
||||||
|
+ this.isBoolean()
|
||||||
|
+ "."
|
||||||
|
+ this.getBENT()
|
||||||
|
+ " "
|
||||||
|
+ this.getX();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Obj obj = new Obj("A beany object", 42, true);
|
||||||
|
|
||||||
|
try {
|
||||||
|
|
||||||
|
s =
|
||||||
|
"{ \"entity\": { \"imageURL\": \"\", \"name\": \"IXXXXXXXXXXXXX\", \"id\": 12336, \"ratingCount\": null, \"averageRating\": null } }";
|
||||||
|
j = new JSONObject(s);
|
||||||
|
System.out.println(j.toString(2));
|
||||||
|
|
||||||
|
jj = new JSONStringer();
|
||||||
|
s =
|
||||||
|
jj.object()
|
||||||
|
.key("single")
|
||||||
|
.value("MARIE HAA'S")
|
||||||
|
.key("Johnny")
|
||||||
|
.value("MARIE HAA\\'S")
|
||||||
|
.key("foo")
|
||||||
|
.value("bar")
|
||||||
|
.key("baz")
|
||||||
|
.array()
|
||||||
|
.object()
|
||||||
|
.key("quux")
|
||||||
|
.value("Thanks, Josh!")
|
||||||
|
.endObject()
|
||||||
|
.endArray()
|
||||||
|
.key("obj keys")
|
||||||
|
.value(JSONObject.getNames(obj))
|
||||||
|
.endObject()
|
||||||
|
.toString();
|
||||||
|
System.out.println(s);
|
||||||
|
|
||||||
|
System.out.println(
|
||||||
|
new JSONStringer()
|
||||||
|
.object()
|
||||||
|
.key("a")
|
||||||
|
.array()
|
||||||
|
.array()
|
||||||
|
.array()
|
||||||
|
.value("b")
|
||||||
|
.endArray()
|
||||||
|
.endArray()
|
||||||
|
.endArray()
|
||||||
|
.endObject()
|
||||||
|
.toString());
|
||||||
|
|
||||||
|
jj = new JSONStringer();
|
||||||
|
jj.array();
|
||||||
|
jj.value(1);
|
||||||
|
jj.array();
|
||||||
|
jj.value(null);
|
||||||
|
jj.array();
|
||||||
|
jj.object();
|
||||||
|
jj.key("empty-array").array().endArray();
|
||||||
|
jj.key("answer").value(42);
|
||||||
|
jj.key("null").value(null);
|
||||||
|
jj.key("false").value(false);
|
||||||
|
jj.key("true").value(true);
|
||||||
|
jj.key("big").value(123456789e+88);
|
||||||
|
jj.key("small").value(123456789e-88);
|
||||||
|
jj.key("empty-object").object().endObject();
|
||||||
|
jj.key("long");
|
||||||
|
jj.value(9223372036854775807L);
|
||||||
|
jj.endObject();
|
||||||
|
jj.value("two");
|
||||||
|
jj.endArray();
|
||||||
|
jj.value(true);
|
||||||
|
jj.endArray();
|
||||||
|
jj.value(98.6);
|
||||||
|
jj.value(-100.0);
|
||||||
|
jj.object();
|
||||||
|
jj.endObject();
|
||||||
|
jj.object();
|
||||||
|
jj.key("one");
|
||||||
|
jj.value(1.00);
|
||||||
|
jj.endObject();
|
||||||
|
jj.value(obj);
|
||||||
|
jj.endArray();
|
||||||
|
System.out.println(jj.toString());
|
||||||
|
|
||||||
|
System.out.println(new JSONArray(jj.toString()).toString(4));
|
||||||
|
|
||||||
|
int ar[] = {1, 2, 3};
|
||||||
|
JSONArray ja = new JSONArray(ar);
|
||||||
|
System.out.println(ja.toString());
|
||||||
|
|
||||||
|
String sa[] = {"aString", "aNumber", "aBoolean"};
|
||||||
|
j = new JSONObject(obj, sa);
|
||||||
|
j.put("Testing JSONString interface", obj);
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{slashes: '///', closetag: '</script>', backslash:'\\\\', ei: {quotes: '\"\\''},eo: {a: '\"quoted\"', b:\"don't\"}, quotes: [\"'\", '\"']}");
|
||||||
|
System.out.println(j.toString(2));
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{foo: [true, false,9876543210, 0.0, 1.00000001, 1.000000000001, 1.00000000000000001,"
|
||||||
|
+ " .00000000000000001, 2.00, 0.1, 2e100, -32,[],{}, \"string\"], "
|
||||||
|
+ " to : null, op : 'Good',"
|
||||||
|
+ "ten:10} postfix comment");
|
||||||
|
j.put("String", "98.6");
|
||||||
|
j.put("JSONObject", new JSONObject());
|
||||||
|
j.put("JSONArray", new JSONArray());
|
||||||
|
j.put("int", 57);
|
||||||
|
j.put("double", 123456789012345678901234567890.);
|
||||||
|
j.put("true", true);
|
||||||
|
j.put("false", false);
|
||||||
|
j.put("null", JSONObject.NULL);
|
||||||
|
j.put("bool", "true");
|
||||||
|
j.put("zero", -0.0);
|
||||||
|
j.put("\\u2028", "\u2028");
|
||||||
|
j.put("\\u2029", "\u2029");
|
||||||
|
a = j.getJSONArray("foo");
|
||||||
|
a.put(666);
|
||||||
|
a.put(2001.99);
|
||||||
|
a.put("so \"fine\".");
|
||||||
|
a.put("so <fine>.");
|
||||||
|
a.put(true);
|
||||||
|
a.put(false);
|
||||||
|
a.put(new JSONArray());
|
||||||
|
a.put(new JSONObject());
|
||||||
|
j.put("keys", JSONObject.getNames(j));
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
System.out.println("String: " + j.getDouble("String"));
|
||||||
|
System.out.println(" bool: " + j.getBoolean("bool"));
|
||||||
|
System.out.println(" to: " + j.getString("to"));
|
||||||
|
System.out.println(" true: " + j.getString("true"));
|
||||||
|
System.out.println(" foo: " + j.getJSONArray("foo"));
|
||||||
|
System.out.println(" op: " + j.getString("op"));
|
||||||
|
System.out.println(" ten: " + j.getInt("ten"));
|
||||||
|
System.out.println(" oops: " + j.optBoolean("oops"));
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{nix: null, nux: false, null: 'null', 'Request-URI': '/', Method: 'GET', 'HTTP-Version': 'HTTP/1.0'}");
|
||||||
|
System.out.println(j.toString(2));
|
||||||
|
System.out.println("isNull: " + j.isNull("nix"));
|
||||||
|
System.out.println(" has: " + j.has("nix"));
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{Envelope: {Body: {\"ns1:doGoogleSearch\": {oe: \"latin1\", filter: true, q: \"'+search+'\", key: \"GOOGLEKEY\", maxResults: 10, \"SOAP-ENV:encodingStyle\": \"http://schemas.xmlsoap.org/soap/encoding/\", start: 0, ie: \"latin1\", safeSearch:false, \"xmlns:ns1\": \"urn:GoogleSearch\"}}}}");
|
||||||
|
System.out.println(j.toString(2));
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{script: 'It is not allowed in HTML to send a close script tag in a string<script>because it confuses browsers</script>so we insert a backslash before the /'}");
|
||||||
|
System.out.println(j.toString());
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
JSONTokener jt =
|
||||||
|
new JSONTokener("{op:'test', to:'session', pre:1}{op:'test', to:'session', pre:2}");
|
||||||
|
j = new JSONObject(jt);
|
||||||
|
System.out.println(j.toString());
|
||||||
|
System.out.println("pre: " + j.optInt("pre"));
|
||||||
|
int i = jt.skipTo('{');
|
||||||
|
System.out.println(i);
|
||||||
|
j = new JSONObject(jt);
|
||||||
|
System.out.println(j.toString());
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
a = new JSONArray(" [\"<escape>\", next is an implied null , , ok,] ");
|
||||||
|
System.out.println(a.toString());
|
||||||
|
System.out.println("");
|
||||||
|
System.out.println("");
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{ fun => with non-standard forms ; forgiving => This package can be used to parse formats that are similar to but not stricting conforming to JSON; why=To make it easier to migrate existing data to JSON,one = [[1.00]]; uno=[[{1=>1}]];'+':+6e66 ;pluses=+++;empty = '' , 'double':0.666,true: TRUE, false: FALSE, null=NULL;[true] = [[!,@;*]]; string=> o. k. ; \r oct=0666; hex=0x666; dec=666; o=0999; noh=0x0x}");
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
System.out.println("");
|
||||||
|
if (j.getBoolean("true") && !j.getBoolean("false")) {
|
||||||
|
System.out.println("It's all good");
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println("");
|
||||||
|
j = new JSONObject(j, new String[] {"dec", "oct", "hex", "missing"});
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
System.out.println("");
|
||||||
|
System.out.println(new JSONStringer().array().value(a).value(j).endArray());
|
||||||
|
|
||||||
|
j =
|
||||||
|
new JSONObject(
|
||||||
|
"{string: \"98.6\", long: 2147483648, int: 2147483647, longer: 9223372036854775807, double: 9223372036854775808}");
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
System.out.println("\ngetInt");
|
||||||
|
System.out.println("int " + j.getInt("int"));
|
||||||
|
System.out.println("long " + j.getInt("long"));
|
||||||
|
System.out.println("longer " + j.getInt("longer"));
|
||||||
|
System.out.println("double " + j.getInt("double"));
|
||||||
|
System.out.println("string " + j.getInt("string"));
|
||||||
|
|
||||||
|
System.out.println("\ngetLong");
|
||||||
|
System.out.println("int " + j.getLong("int"));
|
||||||
|
System.out.println("long " + j.getLong("long"));
|
||||||
|
System.out.println("longer " + j.getLong("longer"));
|
||||||
|
System.out.println("double " + j.getLong("double"));
|
||||||
|
System.out.println("string " + j.getLong("string"));
|
||||||
|
|
||||||
|
System.out.println("\ngetDouble");
|
||||||
|
System.out.println("int " + j.getDouble("int"));
|
||||||
|
System.out.println("long " + j.getDouble("long"));
|
||||||
|
System.out.println("longer " + j.getDouble("longer"));
|
||||||
|
System.out.println("double " + j.getDouble("double"));
|
||||||
|
System.out.println("string " + j.getDouble("string"));
|
||||||
|
|
||||||
|
j.put("good sized", 9223372036854775807L);
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
a = new JSONArray("[2147483647, 2147483648, 9223372036854775807, 9223372036854775808]");
|
||||||
|
System.out.println(a.toString(4));
|
||||||
|
|
||||||
|
System.out.println("\nKeys: ");
|
||||||
|
it = j.keys();
|
||||||
|
while (it.hasNext()) {
|
||||||
|
s = (String) it.next();
|
||||||
|
System.out.println(s + ": " + j.getString(s));
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.println("\naccumulate: ");
|
||||||
|
j = new JSONObject();
|
||||||
|
j.accumulate("stooge", "Curly");
|
||||||
|
j.accumulate("stooge", "Larry");
|
||||||
|
j.accumulate("stooge", "Moe");
|
||||||
|
a = j.getJSONArray("stooge");
|
||||||
|
a.put(5, "Shemp");
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
System.out.println("\nwrite:");
|
||||||
|
System.out.println(j.write(new StringWriter()));
|
||||||
|
|
||||||
|
Collection<?> c = null;
|
||||||
|
Map<?, ?> m = null;
|
||||||
|
|
||||||
|
j = new JSONObject(m);
|
||||||
|
a = new JSONArray(c);
|
||||||
|
j.append("stooge", "Joe DeRita");
|
||||||
|
j.append("stooge", "Shemp");
|
||||||
|
j.accumulate("stooges", "Curly");
|
||||||
|
j.accumulate("stooges", "Larry");
|
||||||
|
j.accumulate("stooges", "Moe");
|
||||||
|
j.accumulate("stoogearray", j.get("stooges"));
|
||||||
|
j.put("map", m);
|
||||||
|
j.put("collection", c);
|
||||||
|
j.put("array", a);
|
||||||
|
a.put(m);
|
||||||
|
a.put(c);
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
s =
|
||||||
|
"{plist=Apple; AnimalSmells = { pig = piggish; lamb = lambish; worm = wormy; }; AnimalSounds = { pig = oink; lamb = baa; worm = baa; Lisa = \"Why is the worm talking like a lamb?\" } ; AnimalColors = { pig = pink; lamb = black; worm = pink; } } ";
|
||||||
|
j = new JSONObject(s);
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
|
||||||
|
s = " (\"San Francisco\", \"New York\", \"Seoul\", \"London\", \"Seattle\", \"Shanghai\")";
|
||||||
|
a = new JSONArray(s);
|
||||||
|
System.out.println(a.toString());
|
||||||
|
|
||||||
|
System.out.println("\nTesting Exceptions: ");
|
||||||
|
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
a = new JSONArray();
|
||||||
|
a.put(Double.NEGATIVE_INFINITY);
|
||||||
|
a.put(Double.NaN);
|
||||||
|
System.out.println(a.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(j.getDouble("stooge"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(j.getDouble("howard"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(j.put(null, "howard"));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(a.getDouble(0));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(a.get(-1));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
System.out.println(a.put(Double.NaN));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
ja = new JSONArray(new Object());
|
||||||
|
System.out.println(ja.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
s = "[)";
|
||||||
|
a = new JSONArray(s);
|
||||||
|
System.out.println(a.toString());
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
s = "{\"koda\": true, \"koda\": true}";
|
||||||
|
j = new JSONObject(s);
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
System.out.print("Exception: ");
|
||||||
|
try {
|
||||||
|
jj = new JSONStringer();
|
||||||
|
s =
|
||||||
|
jj.object()
|
||||||
|
.key("bosanda")
|
||||||
|
.value("MARIE HAA'S")
|
||||||
|
.key("bosanda")
|
||||||
|
.value("MARIE HAA\\'S")
|
||||||
|
.endObject()
|
||||||
|
.toString();
|
||||||
|
System.out.println(j.toString(4));
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println(e.toString());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - AbstractJSONValidationTestCase.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.types.JSONSchemaType;
|
||||||
|
import com.controller.util.FileUtil;
|
||||||
|
import java.io.File;
|
||||||
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
|
public abstract class AbstractJSONValidationTestCase extends TestCase {
|
||||||
|
|
||||||
|
private static final String SAMPLE_OUTPUT_PATH = "sample_output";
|
||||||
|
private static final String SAMPLE_CONFIG_PATH = "config";
|
||||||
|
|
||||||
|
protected String dbName;
|
||||||
|
|
||||||
|
protected void setUp(String dbName) throws Exception {
|
||||||
|
super.setUp();
|
||||||
|
this.dbName = dbName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testJsonKnobs() {
|
||||||
|
String jsonKnobsPath = FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "knobs.json");
|
||||||
|
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonKnobsPath)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testJsonMetrics() {
|
||||||
|
String jsonMetricsBeforePath =
|
||||||
|
FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "metrics_before.json");
|
||||||
|
String jsonMetricsAfterPath =
|
||||||
|
FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "metrics_after.json");
|
||||||
|
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonMetricsBeforePath)));
|
||||||
|
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, new File(jsonMetricsAfterPath)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testJsonSummary() {
|
||||||
|
String jsonSummaryPath = FileUtil.joinPath(SAMPLE_OUTPUT_PATH, this.dbName, "summary.json");
|
||||||
|
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.SUMMARY, new File(jsonSummaryPath)));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void testJsonConfig() {
|
||||||
|
String jsonConfigPath =
|
||||||
|
FileUtil.joinPath(SAMPLE_CONFIG_PATH, "sample_" + this.dbName + "_config.json");
|
||||||
|
assertTrue(JSONSchemaType.isValidJson(JSONSchemaType.CONFIG, new File(jsonConfigPath)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,42 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - TestInvalidJSON.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
import com.controller.types.JSONSchemaType;
|
||||||
|
import junit.framework.TestCase;
|
||||||
|
|
||||||
|
public class TestInvalidJSON extends TestCase {
|
||||||
|
|
||||||
|
// Wrong number of levels for "global"
|
||||||
|
private static final String BAD_JSON_TEXT_1 =
|
||||||
|
"{"
|
||||||
|
+ " \"global\" : {"
|
||||||
|
+ " \"global\" : {"
|
||||||
|
+ " \"auto_generate_certs\": {"
|
||||||
|
+ " \"auto_pram\" : \"NO\""
|
||||||
|
+ " }"
|
||||||
|
+ " }"
|
||||||
|
+ " },"
|
||||||
|
+ " \"local\" : {"
|
||||||
|
+ " }"
|
||||||
|
+ "}";
|
||||||
|
|
||||||
|
// Lacking "local"
|
||||||
|
private static final String BAD_JSON_TEXT_2 =
|
||||||
|
"{"
|
||||||
|
+ " \"global\" : {"
|
||||||
|
+ " \"global1\" : {"
|
||||||
|
+ " \"auto_generate_certs\": \"ON\""
|
||||||
|
+ " }"
|
||||||
|
+ " }"
|
||||||
|
+ "}";
|
||||||
|
|
||||||
|
public void testBadJSONOutput() {
|
||||||
|
assertFalse(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, BAD_JSON_TEXT_1));
|
||||||
|
assertFalse(JSONSchemaType.isValidJson(JSONSchemaType.OUTPUT, BAD_JSON_TEXT_2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - TestMySQLJSON.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
public class TestMySQLJSON extends AbstractJSONValidationTestCase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setUp() throws Exception {
|
||||||
|
super.setUp("mysql");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - TestOracleJSON.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
public class TestOracleJSON extends AbstractJSONValidationTestCase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setUp() throws Exception {
|
||||||
|
super.setUp("oracle");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,15 @@
|
||||||
|
/*
|
||||||
|
* OtterTune - TestPostgresJSON.java
|
||||||
|
*
|
||||||
|
* Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
*/
|
||||||
|
|
||||||
|
package com.controller.collectors;
|
||||||
|
|
||||||
|
public class TestPostgresJSON extends AbstractJSONValidationTestCase {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
protected void setUp() throws Exception {
|
||||||
|
super.setUp("postgres");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,69 @@
|
||||||
|
#
|
||||||
|
# OtterTune - ConfParser.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Mar 23, 2018
|
||||||
|
@author: Jacky, bohan, Dongsheng
|
||||||
|
'''
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
|
||||||
|
def change_postgres_conf(recommendation, postgresqlconf):
|
||||||
|
lines = postgresqlconf.readlines()
|
||||||
|
settings_idx = lines.index("# Add settings for extensions here\n")
|
||||||
|
postgresqlconf.seek(0)
|
||||||
|
postgresqlconf.truncate(0)
|
||||||
|
|
||||||
|
lines = lines[0:(settings_idx + 1)]
|
||||||
|
for line in lines:
|
||||||
|
postgresqlconf.write(line)
|
||||||
|
|
||||||
|
for (knob_name, knob_value) in list(recommendation.items()):
|
||||||
|
postgresqlconf.write(str(knob_name) + " = " + str(knob_value) + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def change_oracle_conf(recommendation, oracle_conf):
|
||||||
|
lines = oracle_conf.readlines()
|
||||||
|
signal = "# configurations recommended by ottertune:\n"
|
||||||
|
if signal not in lines:
|
||||||
|
oracle_conf.write('\n' + signal)
|
||||||
|
oracle_conf.flush()
|
||||||
|
oracle_conf.seek(0)
|
||||||
|
lines = oracle_conf.readlines()
|
||||||
|
settings_idx = lines.index(signal)
|
||||||
|
|
||||||
|
oracle_conf.seek(0)
|
||||||
|
oracle_conf.truncate(0)
|
||||||
|
|
||||||
|
lines = lines[0:(settings_idx + 1)]
|
||||||
|
for line in lines:
|
||||||
|
oracle_conf.write(line)
|
||||||
|
|
||||||
|
for (knob_name, knob_value) in list(recommendation.items()):
|
||||||
|
oracle_conf.write(str(knob_name) + " = " + str(knob_value).strip('B') + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if len(sys.argv) != 4:
|
||||||
|
raise Exception("Usage: python [DB type] ConfParser.py [Next Config] [Current Config]")
|
||||||
|
database_type = sys.argv[1]
|
||||||
|
next_config_name = sys.argv[2]
|
||||||
|
cur_config_name = sys.argv[3]
|
||||||
|
with open(next_config_name, 'r') as next_config, open(cur_config_name, 'r+') as cur_config:
|
||||||
|
config = json.load(next_config, encoding="UTF-8", object_pairs_hook=OrderedDict)
|
||||||
|
recommendation = config['recommendation']
|
||||||
|
if database_type == 'postgres':
|
||||||
|
change_postgres_conf(recommendation, cur_config)
|
||||||
|
elif database_type == 'oracle':
|
||||||
|
change_oracle_conf(recommendation, cur_config)
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(database_type))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,43 @@
|
||||||
|
#
|
||||||
|
# OtterTune - LatencyUDF.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
if (len(sys.argv) != 2):
|
||||||
|
raise Exception("Usage: python udf.py [Output Directory]")
|
||||||
|
|
||||||
|
with open(sys.argv[1] + "/summary.json", "r") as f:
|
||||||
|
conf = json.load(f,
|
||||||
|
encoding="UTF-8",
|
||||||
|
object_pairs_hook=OrderedDict)
|
||||||
|
start_time = conf['start_time']
|
||||||
|
end_time = conf['end_time']
|
||||||
|
|
||||||
|
with open(sys.argv[1] + "/metrics_before.json", "r") as f:
|
||||||
|
conf_before = json.load(f,
|
||||||
|
encoding="UTF-8",
|
||||||
|
object_pairs_hook=OrderedDict)
|
||||||
|
conf_before['global']['udf'] = OrderedDict([("latency", "0")])
|
||||||
|
|
||||||
|
with open(sys.argv[1] + "/metrics_after.json", "r") as f:
|
||||||
|
conf_after = json.load(f,
|
||||||
|
encoding="UTF-8",
|
||||||
|
object_pairs_hook=OrderedDict)
|
||||||
|
conf_after['global']['udf'] = OrderedDict([("latency", str(end_time - start_time))])
|
||||||
|
|
||||||
|
with open(sys.argv[1] + "/metrics_before.json", "w") as f:
|
||||||
|
f.write(json.dumps(conf_before, indent=4))
|
||||||
|
|
||||||
|
with open(sys.argv[1] + "/metrics_after.json", "w") as f:
|
||||||
|
f.write(json.dumps(conf_after, indent=4))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
{
|
||||||
|
"database_type" : "postgres",
|
||||||
|
"database_name" : "tpcc",
|
||||||
|
"database_disk": "/dev/xvda1",
|
||||||
|
"database_conf": "/etc/postgresql/9.6/main/postgresql.conf",
|
||||||
|
"database_save_path": "/home/ubuntu/ottertune",
|
||||||
|
"username" : "bohan",
|
||||||
|
"password" : "bohan",
|
||||||
|
"oltpbench_home": "/home/ubuntu/oltpbench",
|
||||||
|
"oltpbench_config": "/home/ubuntu/oltpbench/config/tpcc_config_postgres.xml",
|
||||||
|
"oltpbench_workload": "tpcc",
|
||||||
|
"oltpbench_log" : "/home/ubuntu/ottertune/client/driver/oltp.log",
|
||||||
|
"controller_config": "/home/ubuntu/ottertune/client/controller/config/sample_postgres_config.json",
|
||||||
|
"controller_log" : "/home/ubuntu/ottertune/client/driver/controller.log",
|
||||||
|
"save_path": "/home/ubuntu/results",
|
||||||
|
"upload_url" : "http://127.0.0.1:8000",
|
||||||
|
"upload_code" : "I5I10PXK3PK27FM86YYS",
|
||||||
|
"lhs_knob_path" : "/home/ubuntu/ottertune/client/driver/knobs/postgres-96.json",
|
||||||
|
"lhs_save_path" : "/home/ubuntu/ottertune/client/driver/configs",
|
||||||
|
"oracle_awr_enabled": false
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,421 @@
|
||||||
|
#
|
||||||
|
# OtterTune - fabfile.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Mar 23, 2018
|
||||||
|
|
||||||
|
@author: bohan
|
||||||
|
'''
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
import os.path
|
||||||
|
import re
|
||||||
|
import glob
|
||||||
|
from multiprocessing import Process
|
||||||
|
from fabric.api import (env, local, task, lcd)
|
||||||
|
from fabric.state import output as fabric_output
|
||||||
|
|
||||||
|
LOG = logging.getLogger()
|
||||||
|
LOG.setLevel(logging.DEBUG)
|
||||||
|
Formatter = logging.Formatter( # pylint: disable=invalid-name
|
||||||
|
"%(asctime)s [%(levelname)s] %(message)s")
|
||||||
|
|
||||||
|
# print the log
|
||||||
|
ConsoleHandler = logging.StreamHandler(sys.stdout) # pylint: disable=invalid-name
|
||||||
|
ConsoleHandler.setFormatter(Formatter)
|
||||||
|
LOG.addHandler(ConsoleHandler)
|
||||||
|
|
||||||
|
# Fabric environment settings
|
||||||
|
env.hosts = ['localhost']
|
||||||
|
fabric_output.update({
|
||||||
|
'running': True,
|
||||||
|
'stdout': True,
|
||||||
|
})
|
||||||
|
|
||||||
|
# intervals of restoring the databse
|
||||||
|
RELOAD_INTERVAL = 10
|
||||||
|
# maximum disk usage
|
||||||
|
MAX_DISK_USAGE = 90
|
||||||
|
|
||||||
|
with open('driver_config.json', 'r') as f:
|
||||||
|
CONF = json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def check_disk_usage():
|
||||||
|
partition = CONF['database_disk']
|
||||||
|
disk_use = 0
|
||||||
|
cmd = "df -h {}".format(partition)
|
||||||
|
out = local(cmd, capture=True).splitlines()[1]
|
||||||
|
m = re.search('\d+(?=%)', out) # pylint: disable=anomalous-backslash-in-string
|
||||||
|
if m:
|
||||||
|
disk_use = int(m.group(0))
|
||||||
|
LOG.info("Current Disk Usage: %s%s", disk_use, '%')
|
||||||
|
return disk_use
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def check_memory_usage():
|
||||||
|
cmd = 'free -m -h'
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def restart_database():
|
||||||
|
if CONF['database_type'] == 'postgres':
|
||||||
|
cmd = 'sudo service postgresql restart'
|
||||||
|
elif CONF['database_type'] == 'oracle':
|
||||||
|
cmd = 'sh oracleScripts/shutdownOracle.sh && sh oracleScripts/startupOracle.sh'
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def drop_database():
|
||||||
|
if CONF['database_type'] == 'postgres':
|
||||||
|
cmd = "PGPASSWORD={} dropdb -e --if-exists {} -U {}".\
|
||||||
|
format(CONF['password'], CONF['database_name'], CONF['username'])
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def create_database():
|
||||||
|
if CONF['database_type'] == 'postgres':
|
||||||
|
cmd = "PGPASSWORD={} createdb -e {} -U {}".\
|
||||||
|
format(CONF['password'], CONF['database_name'], CONF['username'])
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def change_conf():
|
||||||
|
next_conf = 'next_config'
|
||||||
|
cmd = "sudo python3 ConfParser.py {} {} {}".\
|
||||||
|
format(CONF['database_type'], next_conf, CONF['database_conf'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def load_oltpbench():
|
||||||
|
cmd = "./oltpbenchmark -b {} -c {} --create=true --load=true".\
|
||||||
|
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
|
||||||
|
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def run_oltpbench():
|
||||||
|
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile".\
|
||||||
|
format(CONF['oltpbench_workload'], CONF['oltpbench_config'])
|
||||||
|
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def run_oltpbench_bg():
|
||||||
|
cmd = "./oltpbenchmark -b {} -c {} --execute=true -s 5 -o outputfile > {} 2>&1 &".\
|
||||||
|
format(CONF['oltpbench_workload'], CONF['oltpbench_config'], CONF['oltpbench_log'])
|
||||||
|
with lcd(CONF['oltpbench_home']): # pylint: disable=not-context-manager
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def run_controller():
|
||||||
|
cmd = 'sudo gradle run -PappArgs="-c {} -d output/" --no-daemon > {}'.\
|
||||||
|
format(CONF['controller_config'], CONF['controller_log'])
|
||||||
|
with lcd("../controller"): # pylint: disable=not-context-manager
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def signal_controller():
|
||||||
|
pid = int(open('../controller/pid.txt').read())
|
||||||
|
cmd = 'sudo kill -2 {}'.format(pid)
|
||||||
|
with lcd("../controller"): # pylint: disable=not-context-manager
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def save_dbms_result():
|
||||||
|
t = int(time.time())
|
||||||
|
files = ['knobs.json', 'metrics_after.json', 'metrics_before.json', 'summary.json']
|
||||||
|
for f_ in files:
|
||||||
|
f_prefix = f_.split('.')[0]
|
||||||
|
cmd = 'cp ../controller/output/{} {}/{}__{}.json'.\
|
||||||
|
format(f_, CONF['save_path'], t, f_prefix)
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def free_cache():
|
||||||
|
cmd = 'sync; sudo bash -c "echo 1 > /proc/sys/vm/drop_caches"'
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def upload_result():
|
||||||
|
cmd = 'python3 ../../server/website/script/upload/upload.py \
|
||||||
|
../controller/output/ {} {}/new_result/'.format(CONF['upload_code'],
|
||||||
|
CONF['upload_url'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def get_result():
|
||||||
|
cmd = 'python3 ../../script/query_and_get.py {} {} 5'.\
|
||||||
|
format(CONF['upload_url'], CONF['upload_code'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def add_udf():
|
||||||
|
cmd = 'sudo python3 ./LatencyUDF.py ../controller/output/'
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def upload_batch():
|
||||||
|
cmd = 'python3 ./upload_batch.py {} {} {}/new_result/'.format(CONF['save_path'],
|
||||||
|
CONF['upload_code'],
|
||||||
|
CONF['upload_url'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def dump_database():
|
||||||
|
db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
|
||||||
|
if os.path.exists(db_file_path):
|
||||||
|
LOG.info('%s already exists ! ', db_file_path)
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
LOG.info('Dump database %s to %s', CONF['database_name'], db_file_path)
|
||||||
|
# You must create a directory named dpdata through sqlplus in your Oracle database
|
||||||
|
if CONF['database_type'] == 'oracle':
|
||||||
|
cmd = 'expdp {}/{}@{} schemas={} dumpfile={}.dump DIRECTORY=dpdata'.format(
|
||||||
|
'c##tpcc', 'oracle', 'orcldb', 'c##tpcc', 'orcldb')
|
||||||
|
elif CONF['database_type'] == 'postgres':
|
||||||
|
cmd = 'PGPASSWORD={} pg_dump -U {} -F c -d {} > {}'.format(CONF['password'],
|
||||||
|
CONF['username'],
|
||||||
|
CONF['database_name'],
|
||||||
|
db_file_path)
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||||
|
local(cmd)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def restore_database():
|
||||||
|
if CONF['database_type'] == 'oracle':
|
||||||
|
# You must create a directory named dpdata through sqlplus in your Oracle database
|
||||||
|
# The following script assumes such directory exists.
|
||||||
|
# You may want to modify the username, password, and dump file name in the script
|
||||||
|
cmd = 'sh oracleScripts/restoreOracle.sh'
|
||||||
|
elif CONF['database_type'] == 'postgres':
|
||||||
|
db_file_path = '{}/{}.dump'.format(CONF['database_save_path'], CONF['database_name'])
|
||||||
|
drop_database()
|
||||||
|
create_database()
|
||||||
|
cmd = 'PGPASSWORD={} pg_restore -U {} -n public -j 8 -F c -d {} {}'.\
|
||||||
|
format(CONF['password'], CONF['username'], CONF['database_name'], db_file_path)
|
||||||
|
else:
|
||||||
|
raise Exception("Database Type {} Not Implemented !".format(CONF['database_type']))
|
||||||
|
LOG.info('Start restoring database')
|
||||||
|
local(cmd)
|
||||||
|
LOG.info('Finish restoring database')
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_to_start_oltpbench():
|
||||||
|
return (os.path.exists(CONF['controller_log']) and
|
||||||
|
'Output the process pid to'
|
||||||
|
in open(CONF['controller_log']).read())
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_to_start_controller():
|
||||||
|
return (os.path.exists(CONF['oltpbench_log']) and
|
||||||
|
'Warmup complete, starting measurements'
|
||||||
|
in open(CONF['oltpbench_log']).read())
|
||||||
|
|
||||||
|
|
||||||
|
def _ready_to_shut_down_controller():
|
||||||
|
pid_file_path = '../controller/pid.txt'
|
||||||
|
return (os.path.exists(pid_file_path) and os.path.exists(CONF['oltpbench_log']) and
|
||||||
|
'Output throughput samples into file' in open(CONF['oltpbench_log']).read())
|
||||||
|
|
||||||
|
|
||||||
|
def clean_logs():
|
||||||
|
# remove oltpbench log
|
||||||
|
cmd = 'rm -f {}'.format(CONF['oltpbench_log'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
# remove controller log
|
||||||
|
cmd = 'rm -f {}'.format(CONF['controller_log'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def lhs_samples(count=10):
|
||||||
|
cmd = 'python3 lhs.py {} {} {}'.format(count, CONF['lhs_knob_path'], CONF['lhs_save_path'])
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def loop():
|
||||||
|
|
||||||
|
# free cache
|
||||||
|
free_cache()
|
||||||
|
|
||||||
|
# remove oltpbench log and controller log
|
||||||
|
clean_logs()
|
||||||
|
|
||||||
|
# restart database
|
||||||
|
restart_database()
|
||||||
|
|
||||||
|
# check disk usage
|
||||||
|
if check_disk_usage() > MAX_DISK_USAGE:
|
||||||
|
LOG.WARN('Exceeds max disk usage %s', MAX_DISK_USAGE)
|
||||||
|
|
||||||
|
# run controller from another process
|
||||||
|
p = Process(target=run_controller, args=())
|
||||||
|
p.start()
|
||||||
|
LOG.info('Run the controller')
|
||||||
|
|
||||||
|
# run oltpbench as a background job
|
||||||
|
while not _ready_to_start_oltpbench():
|
||||||
|
pass
|
||||||
|
run_oltpbench_bg()
|
||||||
|
LOG.info('Run OLTP-Bench')
|
||||||
|
|
||||||
|
# the controller starts the first collection
|
||||||
|
while not _ready_to_start_controller():
|
||||||
|
pass
|
||||||
|
signal_controller()
|
||||||
|
LOG.info('Start the first collection')
|
||||||
|
|
||||||
|
# stop the experiment
|
||||||
|
while not _ready_to_shut_down_controller():
|
||||||
|
pass
|
||||||
|
signal_controller()
|
||||||
|
LOG.info('Start the second collection, shut down the controller')
|
||||||
|
|
||||||
|
p.join()
|
||||||
|
|
||||||
|
# add user defined target objective
|
||||||
|
# add_udf()
|
||||||
|
|
||||||
|
# save result
|
||||||
|
save_dbms_result()
|
||||||
|
|
||||||
|
# upload result
|
||||||
|
upload_result()
|
||||||
|
|
||||||
|
# get result
|
||||||
|
get_result()
|
||||||
|
|
||||||
|
# change config
|
||||||
|
change_conf()
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def run_lhs():
|
||||||
|
datadir = CONF['lhs_save_path']
|
||||||
|
samples = glob.glob(os.path.join(datadir, 'config_*'))
|
||||||
|
|
||||||
|
# dump database if it's not done before.
|
||||||
|
dump = dump_database()
|
||||||
|
|
||||||
|
for i, sample in enumerate(samples):
|
||||||
|
# reload database periodically
|
||||||
|
if RELOAD_INTERVAL > 0:
|
||||||
|
if i % RELOAD_INTERVAL == 0:
|
||||||
|
if i == 0 and dump is False:
|
||||||
|
restore_database()
|
||||||
|
elif i > 0:
|
||||||
|
restore_database()
|
||||||
|
# free cache
|
||||||
|
free_cache()
|
||||||
|
|
||||||
|
LOG.info('\n\n Start %s-th sample %s \n\n', i, sample)
|
||||||
|
# check memory usage
|
||||||
|
# check_memory_usage()
|
||||||
|
|
||||||
|
# check disk usage
|
||||||
|
if check_disk_usage() > MAX_DISK_USAGE:
|
||||||
|
LOG.WARN('Exceeds max disk usage %s', MAX_DISK_USAGE)
|
||||||
|
|
||||||
|
# copy lhs-sampled config to the to-be-used config
|
||||||
|
cmd = 'cp {} next_config'.format(sample)
|
||||||
|
local(cmd)
|
||||||
|
|
||||||
|
# remove oltpbench log and controller log
|
||||||
|
clean_logs()
|
||||||
|
|
||||||
|
# change config
|
||||||
|
change_conf()
|
||||||
|
|
||||||
|
# restart database
|
||||||
|
restart_database()
|
||||||
|
|
||||||
|
if CONF.get('oracle_awr_enabled', False):
|
||||||
|
# create snapshot for oracle AWR report
|
||||||
|
if CONF['database_type'] == 'oracle':
|
||||||
|
local('sh snapshotOracle.sh')
|
||||||
|
|
||||||
|
# run controller from another process
|
||||||
|
p = Process(target=run_controller, args=())
|
||||||
|
p.start()
|
||||||
|
|
||||||
|
# run oltpbench as a background job
|
||||||
|
while not _ready_to_start_oltpbench():
|
||||||
|
pass
|
||||||
|
run_oltpbench_bg()
|
||||||
|
LOG.info('Run OLTP-Bench')
|
||||||
|
|
||||||
|
while not _ready_to_start_controller():
|
||||||
|
pass
|
||||||
|
signal_controller()
|
||||||
|
LOG.info('Start the first collection')
|
||||||
|
|
||||||
|
while not _ready_to_shut_down_controller():
|
||||||
|
pass
|
||||||
|
# stop the experiment
|
||||||
|
signal_controller()
|
||||||
|
LOG.info('Start the second collection, shut down the controller')
|
||||||
|
|
||||||
|
p.join()
|
||||||
|
|
||||||
|
# save result
|
||||||
|
save_dbms_result()
|
||||||
|
|
||||||
|
# upload result
|
||||||
|
upload_result()
|
||||||
|
|
||||||
|
if CONF.get('oracle_awr_enabled', False):
|
||||||
|
# create oracle AWR report for performance analysis
|
||||||
|
if CONF['database_type'] == 'oracle':
|
||||||
|
local('sh oracleScripts/snapshotOracle.sh && sh oracleScripts/awrOracle.sh')
|
||||||
|
|
||||||
|
|
||||||
|
@task
|
||||||
|
def run_loops(max_iter=1):
|
||||||
|
# dump database if it's not done before.
|
||||||
|
dump = dump_database()
|
||||||
|
|
||||||
|
for i in range(int(max_iter)):
|
||||||
|
if RELOAD_INTERVAL > 0:
|
||||||
|
if i % RELOAD_INTERVAL == 0:
|
||||||
|
if i == 0 and dump is False:
|
||||||
|
restore_database()
|
||||||
|
elif i > 0:
|
||||||
|
restore_database()
|
||||||
|
|
||||||
|
LOG.info('The %s-th Loop Starts / Total Loops %s', i + 1, max_iter)
|
||||||
|
loop()
|
||||||
|
LOG.info('The %s-th Loop Ends / Total Loops %s', i + 1, max_iter)
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "SHARED_POOL_SIZE",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "500MB",
|
||||||
|
"maxval": "2500MB"
|
||||||
|
},
|
||||||
|
"default": "1500MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "DB_CACHE_SIZE",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "10GB",
|
||||||
|
"maxval": "24GB"
|
||||||
|
},
|
||||||
|
"default": "14GB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "LOG_BUFFER",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "10MB",
|
||||||
|
"maxval": "1000MB"
|
||||||
|
},
|
||||||
|
"default": "20MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "LARGE_POOL_SIZE",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "10MB",
|
||||||
|
"maxval": "1000MB"
|
||||||
|
},
|
||||||
|
"default": "100MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "STREAMS_POOL_SIZE",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "10MB",
|
||||||
|
"maxval": "1000MB"
|
||||||
|
},
|
||||||
|
"default": "100MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "bitmap_merge_area_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "1000000",
|
||||||
|
"maxval": "20000000"
|
||||||
|
},
|
||||||
|
"default": "1MB",
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "create_bitmap_area_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "1000000",
|
||||||
|
"maxval": "100000000"
|
||||||
|
},
|
||||||
|
"default": "8MB",
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "hash_area_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "65536",
|
||||||
|
"maxval": "1000000"
|
||||||
|
},
|
||||||
|
"default": "65536",
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "sort_area_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "128000",
|
||||||
|
"maxval": "2000000"
|
||||||
|
},
|
||||||
|
"default": "128000",
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,110 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"name": "effective_cache_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "4GB",
|
||||||
|
"maxval": "16GB"
|
||||||
|
},
|
||||||
|
"default": "4GB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "shared_buffers",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "128MB",
|
||||||
|
"maxval": "12GB"
|
||||||
|
},
|
||||||
|
"default": "128MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "max_parallel_workers_per_gather",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": 0,
|
||||||
|
"maxval": 8
|
||||||
|
},
|
||||||
|
"default": 0,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "default_statistics_target",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": 100,
|
||||||
|
"maxval": 2048
|
||||||
|
},
|
||||||
|
"default": 100,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "bgwriter_lru_maxpages",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": 0,
|
||||||
|
"maxval": 1000
|
||||||
|
},
|
||||||
|
"default": 10,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "checkpoint_timeout",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "10ms",
|
||||||
|
"maxval": "1min"
|
||||||
|
},
|
||||||
|
"default": "200ms",
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "random_page_cost",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": 1,
|
||||||
|
"maxval": 10
|
||||||
|
},
|
||||||
|
"default": 4.0,
|
||||||
|
"type": "float"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "checkpoint_completion_target",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": 0.1,
|
||||||
|
"maxval": 0.9
|
||||||
|
},
|
||||||
|
"default": 0.5,
|
||||||
|
"type": "float"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "checkpoint_timeout",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "1min",
|
||||||
|
"maxval": "30min"
|
||||||
|
},
|
||||||
|
"default": "5min",
|
||||||
|
"type": "time"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "max_wal_size",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "256MB",
|
||||||
|
"maxval": "16GB"
|
||||||
|
},
|
||||||
|
"default": "1GB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "temp_buffers",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "8MB",
|
||||||
|
"maxval": "1GB"
|
||||||
|
},
|
||||||
|
"default": "8MB",
|
||||||
|
"type": "bytes"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "work_mem",
|
||||||
|
"tuning_range": {
|
||||||
|
"minval": "4MB",
|
||||||
|
"maxval": "1GB"
|
||||||
|
},
|
||||||
|
"default": "4MB",
|
||||||
|
"type": "bytes"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
@ -0,0 +1,126 @@
|
||||||
|
#
|
||||||
|
# OtterTune - lhs.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import numpy as np
|
||||||
|
from pyDOE import lhs
|
||||||
|
from scipy.stats import uniform
|
||||||
|
from hurry.filesize import size
|
||||||
|
|
||||||
|
|
||||||
|
BYTES_SYSTEM = [
|
||||||
|
(1024 ** 5, 'PB'),
|
||||||
|
(1024 ** 4, 'TB'),
|
||||||
|
(1024 ** 3, 'GB'),
|
||||||
|
(1024 ** 2, 'MB'),
|
||||||
|
(1024 ** 1, 'kB'),
|
||||||
|
(1024 ** 0, 'B'),
|
||||||
|
]
|
||||||
|
|
||||||
|
TIME_SYSTEM = [
|
||||||
|
(1000 * 60 * 60 * 24, 'd'),
|
||||||
|
(1000 * 60 * 60, 'h'),
|
||||||
|
(1000 * 60, 'min'),
|
||||||
|
(1000, 's'),
|
||||||
|
(1, 'ms'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def get_raw_size(value, system):
|
||||||
|
for factor, suffix in system:
|
||||||
|
if value.endswith(suffix):
|
||||||
|
if len(value) == len(suffix):
|
||||||
|
amount = 1
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
amount = int(value[:-len(suffix)])
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return amount * factor
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def get_knob_raw(value, knob_type):
|
||||||
|
if knob_type == 'integer':
|
||||||
|
return int(value)
|
||||||
|
elif knob_type == 'float':
|
||||||
|
return float(value)
|
||||||
|
elif knob_type == 'bytes':
|
||||||
|
return get_raw_size(value, BYTES_SYSTEM)
|
||||||
|
elif knob_type == 'time':
|
||||||
|
return get_raw_size(value, TIME_SYSTEM)
|
||||||
|
else:
|
||||||
|
raise Exception('Knob Type does not support')
|
||||||
|
|
||||||
|
|
||||||
|
def get_knob_readable(value, knob_type):
|
||||||
|
if knob_type == 'integer':
|
||||||
|
return int(round(value))
|
||||||
|
elif knob_type == 'float':
|
||||||
|
return float(value)
|
||||||
|
elif knob_type == 'bytes':
|
||||||
|
value = int(round(value))
|
||||||
|
return size(value, system=BYTES_SYSTEM)
|
||||||
|
elif knob_type == 'time':
|
||||||
|
value = int(round(value))
|
||||||
|
return size(value, system=TIME_SYSTEM)
|
||||||
|
else:
|
||||||
|
raise Exception('Knob Type does not support')
|
||||||
|
|
||||||
|
|
||||||
|
def get_knobs_readable(values, types):
|
||||||
|
result = []
|
||||||
|
for i, value in enumerate(values):
|
||||||
|
result.append(get_knob_readable(value, types[i]))
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main(args):
|
||||||
|
|
||||||
|
if (len(sys.argv) != 4):
|
||||||
|
raise Exception("Usage: python3 lhs.py [Samples Count] [Knob Path] [Save Path]")
|
||||||
|
|
||||||
|
knob_path = args[2]
|
||||||
|
save_path = args[3]
|
||||||
|
with open(knob_path, "r") as f:
|
||||||
|
tuning_knobs = json.load(f)
|
||||||
|
|
||||||
|
names = []
|
||||||
|
maxvals = []
|
||||||
|
minvals = []
|
||||||
|
types = []
|
||||||
|
|
||||||
|
for knob in tuning_knobs:
|
||||||
|
names.append(knob['name'])
|
||||||
|
maxvals.append(get_knob_raw(knob['tuning_range']['maxval'], knob['type']))
|
||||||
|
minvals.append(get_knob_raw(knob['tuning_range']['minval'], knob['type']))
|
||||||
|
types.append(knob['type'])
|
||||||
|
|
||||||
|
nsamples = int(args[1])
|
||||||
|
nfeats = len(tuning_knobs)
|
||||||
|
samples = lhs(nfeats, samples=nsamples, criterion='maximin')
|
||||||
|
maxvals = np.array(maxvals)
|
||||||
|
minvals = np.array(minvals)
|
||||||
|
scales = maxvals - minvals
|
||||||
|
for fidx in range(nfeats):
|
||||||
|
samples[:, fidx] = uniform(loc=minvals[fidx], scale=scales[fidx]).ppf(samples[:, fidx])
|
||||||
|
|
||||||
|
samples_readable = []
|
||||||
|
for sample in samples:
|
||||||
|
samples_readable.append(get_knobs_readable(sample, types))
|
||||||
|
|
||||||
|
config = {'recommendation': {}}
|
||||||
|
for sidx in range(nsamples):
|
||||||
|
for fidx in range(nfeats):
|
||||||
|
config["recommendation"][names[fidx]] = samples_readable[sidx][fidx]
|
||||||
|
with open(os.path.join(save_path, 'config_' + str(sidx)), 'w+') as f:
|
||||||
|
f.write(json.dumps(config))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main(sys.argv)
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
sudo -b nohup fab run_lhs > lhs.log 2>&1
|
||||||
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/bin/sh
|
||||||
|
su - oracle <<EON
|
||||||
|
oracle
|
||||||
|
sqlplus / as sysdba <<EOF
|
||||||
|
@/home/oracle/ottertune/client/driver/autoawr.sql;
|
||||||
|
quit
|
||||||
|
EOF
|
||||||
|
exit
|
||||||
|
EON
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/bin/sh
|
||||||
|
su - oracle <<EON
|
||||||
|
oracle #system password
|
||||||
|
|
||||||
|
sqlplus / as sysdba <<EOF
|
||||||
|
drop user c##tpcc cascade;
|
||||||
|
# username
|
||||||
|
create user c##tpcc identified by oracle;
|
||||||
|
# username password
|
||||||
|
quit
|
||||||
|
EOF
|
||||||
|
|
||||||
|
impdp 'userid="/ as sysdba"' schemas=c##tpcc dumpfile=orcldb.dump DIRECTORY=dpdata
|
||||||
|
# username database_name db_directory
|
||||||
|
|
||||||
|
sqlplus / as sysdba <<EOF #restart the database
|
||||||
|
shutdown immediate
|
||||||
|
startup
|
||||||
|
quit
|
||||||
|
EOF
|
||||||
|
|
||||||
|
exit
|
||||||
|
EON
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/sh
|
||||||
|
su - oracle <<EON
|
||||||
|
oracle
|
||||||
|
|
||||||
|
sqlplus / as sysdba <<EOF
|
||||||
|
shutdown immediate
|
||||||
|
exit
|
||||||
|
EOF
|
||||||
|
|
||||||
|
exit
|
||||||
|
EON
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/sh
|
||||||
|
su - oracle <<EON
|
||||||
|
oracle
|
||||||
|
|
||||||
|
sqlplus / as sysdba <<EOF
|
||||||
|
exec dbms_workload_repository.create_snapshot;
|
||||||
|
quit
|
||||||
|
EOF
|
||||||
|
|
||||||
|
exit
|
||||||
|
EON
|
||||||
|
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/sh
|
||||||
|
su - oracle <<EON
|
||||||
|
oracle
|
||||||
|
|
||||||
|
sqlplus / as sysdba <<EOF
|
||||||
|
startup
|
||||||
|
quit
|
||||||
|
EOF
|
||||||
|
|
||||||
|
exit
|
||||||
|
EON
|
||||||
|
|
@ -0,0 +1,63 @@
|
||||||
|
#
|
||||||
|
# OtterTune - upload_batch.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
import argparse
|
||||||
|
import glob
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
LOG.addHandler(logging.StreamHandler())
|
||||||
|
LOG.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
# Upload all files in the datadir to OtterTune's server side.
|
||||||
|
# You may want to upload your training data to the non-tuning session.
|
||||||
|
def upload_batch(datadir, upload_code, url):
|
||||||
|
|
||||||
|
samples = glob.glob(os.path.join(datadir, '*__summary.json'))
|
||||||
|
count = len(samples)
|
||||||
|
samples_prefix = []
|
||||||
|
|
||||||
|
LOG.info('Uploading %d samples in %s...', count, datadir)
|
||||||
|
for sample in samples:
|
||||||
|
prefix = sample.split('/')[-1].split('__')[0]
|
||||||
|
samples_prefix.append(prefix)
|
||||||
|
|
||||||
|
for i in range(count):
|
||||||
|
prefix = samples_prefix[i]
|
||||||
|
params = {
|
||||||
|
'summary': open(os.path.join(datadir, '{}__summary.json'.format(prefix)), 'rb'),
|
||||||
|
'knobs': open(os.path.join(datadir, '{}__knobs.json'.format(prefix)), 'rb'),
|
||||||
|
'metrics_before': open(os.path.join(datadir,
|
||||||
|
'{}__metrics_before.json'.format(prefix)), 'rb'),
|
||||||
|
'metrics_after': open(os.path.join(datadir,
|
||||||
|
'{}__metrics_after.json'.format(prefix)), 'rb'),
|
||||||
|
}
|
||||||
|
|
||||||
|
LOG.info('Upload %d-th sample %s__*.json', i + 1, prefix)
|
||||||
|
response = requests.post(url,
|
||||||
|
files=params,
|
||||||
|
data={'upload_code': upload_code})
|
||||||
|
LOG.info(response.content)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Upload generated data to the website")
|
||||||
|
parser.add_argument('datadir', type=str, nargs=1,
|
||||||
|
help='Directory containing the generated data')
|
||||||
|
parser.add_argument('upload_code', type=str, nargs=1,
|
||||||
|
help='The website\'s upload code')
|
||||||
|
parser.add_argument('url', type=str, default='http://0.0.0.0:8000/new_result/',
|
||||||
|
nargs='?', help='The upload url: server_ip/new_result/')
|
||||||
|
args = parser.parse_args()
|
||||||
|
upload_batch(args.datadir[0], args.upload_code[0], args.url)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
*.swp
|
||||||
|
|
@ -0,0 +1,23 @@
|
||||||
|
FROM ottertune-base
|
||||||
|
|
||||||
|
ENV DJANGO_SETTINGS_MODULE=website.settings
|
||||||
|
ENV C_FORCE_ROOT=true
|
||||||
|
|
||||||
|
RUN mkdir -p /app
|
||||||
|
|
||||||
|
COPY . /app
|
||||||
|
|
||||||
|
WORKDIR /app/server/website
|
||||||
|
|
||||||
|
COPY ./docker/credentials.py ./website/settings
|
||||||
|
COPY ./docker/start.sh .
|
||||||
|
COPY ./docker/createadmin.py .
|
||||||
|
COPY ./docker/wait-for-it.sh .
|
||||||
|
|
||||||
|
RUN chmod +x ./*.sh
|
||||||
|
|
||||||
|
RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
|
||||||
|
&& mv tmp ./website/settings/common.py
|
||||||
|
|
||||||
|
ENTRYPOINT ["./start.sh"]
|
||||||
|
|
||||||
|
|
@ -0,0 +1,31 @@
|
||||||
|
FROM centos:7
|
||||||
|
|
||||||
|
ARG GRADLE_VERSION=gradle-5.5.1
|
||||||
|
|
||||||
|
ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
|
||||||
|
ENV PATH=${GRADLE_HOME}/bin:${PATH}
|
||||||
|
|
||||||
|
COPY ./server/website/requirements.txt /
|
||||||
|
|
||||||
|
RUN yum update -y \
|
||||||
|
&& yum install -y mariadb mariadb-devel \
|
||||||
|
https://centos7.iuscommunity.org/ius-release.rpm \
|
||||||
|
&& yum install -y gcc git MySQL-python openldap-devel \
|
||||||
|
parallel python36u python36u-devel python36u-libs \
|
||||||
|
python36u-pip python36u-tkinter rabbitmq-server \
|
||||||
|
java-11-openjdk-devel wget which unzip curl \
|
||||||
|
&& yum -y autoremove \
|
||||||
|
&& yum clean metadata \
|
||||||
|
&& yum clean all \
|
||||||
|
&& ln -s `which python3.6` /usr/bin/python3 \
|
||||||
|
&& ln -s `which pip3.6` /usr/bin/pip3 \
|
||||||
|
&& wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
|
||||||
|
&& unzip ${GRADLE_VERSION}-bin.zip -d /opt \
|
||||||
|
&& rm ${GRADLE_VERSION}-bin.zip \
|
||||||
|
&& python3 --version \
|
||||||
|
&& pip3 --version \
|
||||||
|
&& javac --version \
|
||||||
|
&& gradle --version \
|
||||||
|
&& pip3 install -r /requirements.txt \
|
||||||
|
&& rm /requirements.txt
|
||||||
|
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
FROM ubuntu:18.04
|
||||||
|
|
||||||
|
ARG GRADLE_VERSION=gradle-5.5.1
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
ENV GRADLE_HOME=/opt/${GRADLE_VERSION}
|
||||||
|
ENV PATH=${GRADLE_HOME}/bin:${PATH}
|
||||||
|
|
||||||
|
COPY ./server/website/requirements.txt /
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y python3.6 python3-pip python3-tk \
|
||||||
|
mysql-client libmysqlclient-dev python-mysqldb \
|
||||||
|
openjdk-11-jdk git unzip wget curl \
|
||||||
|
&& wget https://services.gradle.org/distributions/${GRADLE_VERSION}-bin.zip \
|
||||||
|
&& unzip ${GRADLE_VERSION}-bin.zip -d /opt \
|
||||||
|
&& rm ${GRADLE_VERSION}-bin.zip \
|
||||||
|
&& python3 --version \
|
||||||
|
&& pip3 --version \
|
||||||
|
&& javac --version \
|
||||||
|
&& gradle --version \
|
||||||
|
&& pip3 install -r /requirements.txt \
|
||||||
|
&& rm /requirements.txt
|
||||||
|
|
||||||
|
|
@ -0,0 +1,21 @@
|
||||||
|
FROM ottertune-base
|
||||||
|
|
||||||
|
ENV DJANGO_SETTINGS_MODULE=website.settings
|
||||||
|
ENV C_FORCE_ROOT=true
|
||||||
|
|
||||||
|
RUN mkdir -p /app
|
||||||
|
|
||||||
|
COPY . /app
|
||||||
|
|
||||||
|
WORKDIR /app/server/website
|
||||||
|
|
||||||
|
RUN pip3 install codecov
|
||||||
|
|
||||||
|
COPY ./docker/credentials.py ./website/settings
|
||||||
|
COPY ./docker/wait-for-it.sh .
|
||||||
|
|
||||||
|
RUN chmod +x ./*.sh
|
||||||
|
|
||||||
|
RUN sed s/'@localhost'/'@rabbitmq'/g ./website/settings/common.py > tmp \
|
||||||
|
&& mv tmp ./website/settings/common.py
|
||||||
|
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
|
import os
|
||||||
|
import django
|
||||||
|
|
||||||
|
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "website.settings")
|
||||||
|
django.setup()
|
||||||
|
|
||||||
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
|
username = os.environ.get('ADMIN_USER', 'admin')
|
||||||
|
password = os.environ.get('ADMIN_PASSWORD')
|
||||||
|
email = os.environ.get('ADMIN_EMAIL', 'admin@example.com')
|
||||||
|
|
||||||
|
if password:
|
||||||
|
if not User.objects.filter(username=username).exists():
|
||||||
|
print(f"Creating '{username}' user...")
|
||||||
|
User.objects.create_superuser(username=username,
|
||||||
|
password=password,
|
||||||
|
email=email)
|
||||||
|
print(f"'{username}' user created!")
|
||||||
|
else:
|
||||||
|
print(f"'{username}' user already exists! Setting '{username}' password")
|
||||||
|
u = User.objects.get(username=username)
|
||||||
|
u.set_password(password)
|
||||||
|
u.save()
|
||||||
|
|
@ -0,0 +1,24 @@
|
||||||
|
import secrets
|
||||||
|
from os import environ as env
|
||||||
|
|
||||||
|
db_user = env.get('MYSQL_USER')
|
||||||
|
db_pwd = env.get('MYSQL_PASSWORD')
|
||||||
|
db_host = env.get('MYSQL_HOST')
|
||||||
|
db_port = env.get('MYSQL_PORT', '3306')
|
||||||
|
debug = env.get('DEBUG')
|
||||||
|
|
||||||
|
SECRET_KEY = secrets.token_hex(16)
|
||||||
|
DATABASES = {
|
||||||
|
'default': {'ENGINE': 'django.db.backends.mysql',
|
||||||
|
'NAME': 'ottertune',
|
||||||
|
'USER': db_user,
|
||||||
|
'PASSWORD': db_pwd,
|
||||||
|
'HOST': db_host,
|
||||||
|
'PORT': db_port,
|
||||||
|
'OPTIONS': {'init_command': 'SET sql_mode=\'STRICT_TRANS_TABLES\',innodb_strict_mode=1',}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
DEBUG = True
|
||||||
|
ADMINS = ()
|
||||||
|
MANAGERS = ADMINS
|
||||||
|
ALLOWED_HOSTS = []
|
||||||
|
|
@ -0,0 +1,85 @@
|
||||||
|
version: "3"
|
||||||
|
services:
|
||||||
|
base:
|
||||||
|
build:
|
||||||
|
context: ../
|
||||||
|
dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
|
||||||
|
image: ottertune-base
|
||||||
|
container_name: ottertune-base
|
||||||
|
labels:
|
||||||
|
NAME: "ottertune-base"
|
||||||
|
|
||||||
|
test:
|
||||||
|
build:
|
||||||
|
context: ../
|
||||||
|
dockerfile: ./docker/Dockerfile.test
|
||||||
|
image: ottertune-test
|
||||||
|
container_name: ottertune-test
|
||||||
|
expose:
|
||||||
|
- "8000"
|
||||||
|
ports:
|
||||||
|
- "8000:8000"
|
||||||
|
links:
|
||||||
|
- mysql
|
||||||
|
- rabbitmq
|
||||||
|
depends_on:
|
||||||
|
- mysql
|
||||||
|
- rabbitmq
|
||||||
|
environment:
|
||||||
|
DEBUG: 'True'
|
||||||
|
MYSQL_USER: 'root'
|
||||||
|
MYSQL_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_HOST: 'mysql'
|
||||||
|
MAX_DB_CONN_ATTEMPTS: 15
|
||||||
|
labels:
|
||||||
|
NAME: "ottertune-test"
|
||||||
|
volumes:
|
||||||
|
- media_data:/app/server/website/media
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
|
||||||
|
mysql:
|
||||||
|
image: mysql:5.7
|
||||||
|
container_name: mysql
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
MYSQL_ROOT_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_DATABASE: 'ottertune'
|
||||||
|
expose:
|
||||||
|
- "3306"
|
||||||
|
ports:
|
||||||
|
- "3306:3306"
|
||||||
|
labels:
|
||||||
|
NAME: "mysql"
|
||||||
|
volumes:
|
||||||
|
- mysql_data:/var/lib/mysql
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
|
||||||
|
rabbitmq:
|
||||||
|
image: "rabbitmq:3-management"
|
||||||
|
container_name: rabbitmq
|
||||||
|
restart: always
|
||||||
|
hostname: "rabbitmq"
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: "guest"
|
||||||
|
RABBITMQ_DEFAULT_PASS: "guest"
|
||||||
|
RABBITMQ_DEFAULT_VHOST: "/"
|
||||||
|
expose:
|
||||||
|
- "15672"
|
||||||
|
- "5672"
|
||||||
|
ports:
|
||||||
|
- "15672:15672"
|
||||||
|
- "5672:5672"
|
||||||
|
labels:
|
||||||
|
NAME: "rabbitmq"
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
volumes:
|
||||||
|
mysql_data:
|
||||||
|
media_data:
|
||||||
|
networks:
|
||||||
|
ottertune-net:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
|
|
@ -0,0 +1,88 @@
|
||||||
|
version: "3"
|
||||||
|
services:
|
||||||
|
base:
|
||||||
|
build:
|
||||||
|
context: ../
|
||||||
|
dockerfile: ./docker/Dockerfile.base-ubuntu-18.04
|
||||||
|
image: ottertune-base
|
||||||
|
container_name: ottertune-base
|
||||||
|
labels:
|
||||||
|
NAME: "ottertune-base"
|
||||||
|
|
||||||
|
web:
|
||||||
|
build:
|
||||||
|
context: ../
|
||||||
|
dockerfile: ./docker/Dockerfile
|
||||||
|
image: ottertune
|
||||||
|
container_name: ottertune
|
||||||
|
expose:
|
||||||
|
- "8000"
|
||||||
|
ports:
|
||||||
|
- "8000:8000"
|
||||||
|
links:
|
||||||
|
- mysql
|
||||||
|
- rabbitmq
|
||||||
|
depends_on:
|
||||||
|
- base
|
||||||
|
- mysql
|
||||||
|
- rabbitmq
|
||||||
|
environment:
|
||||||
|
DEBUG: 'True'
|
||||||
|
ADMIN_PASSWORD: 'changeme'
|
||||||
|
ADMIN_EMAIL: 'admin@example.com'
|
||||||
|
MYSQL_USER: 'root'
|
||||||
|
MYSQL_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_HOST: 'mysql'
|
||||||
|
MAX_DB_CONN_ATTEMPTS: 15
|
||||||
|
labels:
|
||||||
|
NAME: "ottertune"
|
||||||
|
volumes:
|
||||||
|
- media_data:/app/server/website/media
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
|
||||||
|
mysql:
|
||||||
|
image: mysql:5.7
|
||||||
|
container_name: mysql
|
||||||
|
restart: always
|
||||||
|
environment:
|
||||||
|
MYSQL_ROOT_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_PASSWORD: 'ottertune'
|
||||||
|
MYSQL_DATABASE: 'ottertune'
|
||||||
|
expose:
|
||||||
|
- "3306"
|
||||||
|
ports:
|
||||||
|
- "3306:3306"
|
||||||
|
labels:
|
||||||
|
NAME: "mysql"
|
||||||
|
volumes:
|
||||||
|
- mysql_data:/var/lib/mysql
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
|
||||||
|
rabbitmq:
|
||||||
|
image: "rabbitmq:3-management"
|
||||||
|
container_name: rabbitmq
|
||||||
|
restart: always
|
||||||
|
hostname: "rabbitmq"
|
||||||
|
environment:
|
||||||
|
RABBITMQ_DEFAULT_USER: "guest"
|
||||||
|
RABBITMQ_DEFAULT_PASS: "guest"
|
||||||
|
RABBITMQ_DEFAULT_VHOST: "/"
|
||||||
|
expose:
|
||||||
|
- "15672"
|
||||||
|
- "5672"
|
||||||
|
ports:
|
||||||
|
- "15672:15672"
|
||||||
|
- "5672:5672"
|
||||||
|
labels:
|
||||||
|
NAME: "rabbitmq"
|
||||||
|
networks:
|
||||||
|
- ottertune-net
|
||||||
|
volumes:
|
||||||
|
mysql_data:
|
||||||
|
media_data:
|
||||||
|
networks:
|
||||||
|
ottertune-net:
|
||||||
|
driver: bridge
|
||||||
|
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# Wait for MySQL connection
|
||||||
|
/bin/bash wait-for-it.sh
|
||||||
|
|
||||||
|
## Needs a connection to a DB so migrations go here
|
||||||
|
python3 manage.py makemigrations website
|
||||||
|
python3 manage.py migrate
|
||||||
|
python3 createadmin.py
|
||||||
|
|
||||||
|
python3 manage.py celery worker --loglevel=info --pool=threads &
|
||||||
|
python3 manage.py celerybeat --verbosity=2 --loglevel=info &
|
||||||
|
python3 manage.py runserver 0.0.0.0:8000
|
||||||
|
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# wait until MySQL is really available
|
||||||
|
maxcounter=${MAX_DB_CONN_ATTEMPTS:-45}
|
||||||
|
echo "Trying to connect to mysql, max attempts="$maxcounter
|
||||||
|
|
||||||
|
counter=1
|
||||||
|
while ! mysql --host="$MYSQL_HOST" --protocol TCP -u"$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "show databases;" > /dev/null 2>&1; do
|
||||||
|
sleep 1
|
||||||
|
counter=`expr $counter + 1`
|
||||||
|
if [ $counter -gt $maxcounter ]; then
|
||||||
|
>&2 echo "We have been waiting for MySQL too long already; failing."
|
||||||
|
exit 1
|
||||||
|
fi;
|
||||||
|
done
|
||||||
|
echo "-=------------------------------------------------------"
|
||||||
|
echo "-=------------------------------------------------------"
|
||||||
|
echo "Connected to MySQL!"
|
||||||
|
echo "-=------------------------------------------------------"
|
||||||
|
echo "-=------------------------------------------------------"
|
||||||
|
|
@ -0,0 +1,238 @@
|
||||||
|
<?xml version="1.0"?>
|
||||||
|
<!DOCTYPE module PUBLIC
|
||||||
|
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
||||||
|
"http://checkstyle.sourceforge.net/dtds/configuration_1_3.dtd">
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Checkstyle configuration that checks the Google coding conventions from Google Java Style
|
||||||
|
that can be found at https://google.github.io/styleguide/javaguide.html.
|
||||||
|
|
||||||
|
Checkstyle is very configurable. Be sure to read the documentation at
|
||||||
|
http://checkstyle.sf.net (or in your downloaded distribution).
|
||||||
|
|
||||||
|
To completely disable a check, just comment it out or delete it from the file.
|
||||||
|
|
||||||
|
Authors: Max Vetrenko, Ruslan Diachenko, Roman Ivanov.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<module name = "Checker">
|
||||||
|
<property name="charset" value="UTF-8"/>
|
||||||
|
|
||||||
|
<property name="severity" value="warning"/>
|
||||||
|
|
||||||
|
<property name="fileExtensions" value="java, properties, xml"/>
|
||||||
|
<!-- Checks for whitespace -->
|
||||||
|
<!-- See http://checkstyle.sf.net/config_whitespace.html -->
|
||||||
|
<module name="FileTabCharacter">
|
||||||
|
<property name="eachLine" value="true"/>
|
||||||
|
</module>
|
||||||
|
|
||||||
|
<module name="TreeWalker">
|
||||||
|
<module name="OuterTypeFilename"/>
|
||||||
|
<module name="IllegalTokenText">
|
||||||
|
<property name="tokens" value="STRING_LITERAL, CHAR_LITERAL"/>
|
||||||
|
<property name="format" value="\\u00(09|0(a|A)|0(c|C)|0(d|D)|22|27|5(C|c))|\\(0(10|11|12|14|15|42|47)|134)"/>
|
||||||
|
<property name="message" value="Consider using special escape sequence instead of octal value or Unicode escaped value."/>
|
||||||
|
</module>
|
||||||
|
<module name="AvoidEscapedUnicodeCharacters">
|
||||||
|
<property name="allowEscapesForControlCharacters" value="true"/>
|
||||||
|
<property name="allowByTailComment" value="true"/>
|
||||||
|
<property name="allowNonPrintableEscapes" value="true"/>
|
||||||
|
</module>
|
||||||
|
<module name="LineLength">
|
||||||
|
<property name="max" value="100"/>
|
||||||
|
<property name="ignorePattern" value="^package.*|^import.*|a href|href|http://|https://|ftp://"/>
|
||||||
|
</module>
|
||||||
|
<module name="AvoidStarImport"/>
|
||||||
|
<module name="OneTopLevelClass"/>
|
||||||
|
<module name="NoLineWrap"/>
|
||||||
|
<module name="EmptyBlock">
|
||||||
|
<property name="option" value="TEXT"/>
|
||||||
|
<property name="tokens" value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/>
|
||||||
|
</module>
|
||||||
|
<module name="NeedBraces"/>
|
||||||
|
<module name="LeftCurly"/>
|
||||||
|
<module name="RightCurly">
|
||||||
|
<property name="id" value="RightCurlySame"/>
|
||||||
|
<property name="tokens" value="LITERAL_TRY, LITERAL_CATCH, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_DO"/>
|
||||||
|
</module>
|
||||||
|
<module name="RightCurly">
|
||||||
|
<property name="id" value="RightCurlyAlone"/>
|
||||||
|
<property name="option" value="alone"/>
|
||||||
|
<property name="tokens" value="CLASS_DEF, METHOD_DEF, CTOR_DEF, LITERAL_FOR, LITERAL_WHILE, STATIC_INIT, INSTANCE_INIT"/>
|
||||||
|
</module>
|
||||||
|
<module name="WhitespaceAround">
|
||||||
|
<property name="allowEmptyConstructors" value="true"/>
|
||||||
|
<property name="allowEmptyMethods" value="true"/>
|
||||||
|
<property name="allowEmptyTypes" value="true"/>
|
||||||
|
<property name="allowEmptyLoops" value="true"/>
|
||||||
|
<message key="ws.notFollowed"
|
||||||
|
value="WhitespaceAround: ''{0}'' is not followed by whitespace. Empty blocks may only be represented as '{}' when not part of a multi-block statement (4.1.3)"/>
|
||||||
|
<message key="ws.notPreceded"
|
||||||
|
value="WhitespaceAround: ''{0}'' is not preceded with whitespace."/>
|
||||||
|
</module>
|
||||||
|
<module name="OneStatementPerLine"/>
|
||||||
|
<module name="MultipleVariableDeclarations"/>
|
||||||
|
<module name="ArrayTypeStyle"/>
|
||||||
|
<module name="MissingSwitchDefault"/>
|
||||||
|
<module name="FallThrough"/>
|
||||||
|
<module name="UpperEll"/>
|
||||||
|
<module name="ModifierOrder"/>
|
||||||
|
<module name="EmptyLineSeparator">
|
||||||
|
<property name="allowNoEmptyLineBetweenFields" value="true"/>
|
||||||
|
</module>
|
||||||
|
<module name="SeparatorWrap">
|
||||||
|
<property name="id" value="SeparatorWrapDot"/>
|
||||||
|
<property name="tokens" value="DOT"/>
|
||||||
|
<property name="option" value="nl"/>
|
||||||
|
</module>
|
||||||
|
<module name="SeparatorWrap">
|
||||||
|
<property name="id" value="SeparatorWrapComma"/>
|
||||||
|
<property name="tokens" value="COMMA"/>
|
||||||
|
<property name="option" value="EOL"/>
|
||||||
|
</module>
|
||||||
|
<module name="SeparatorWrap">
|
||||||
|
<!-- ELLIPSIS is EOL until https://github.com/google/styleguide/issues/258 -->
|
||||||
|
<property name="id" value="SeparatorWrapEllipsis"/>
|
||||||
|
<property name="tokens" value="ELLIPSIS"/>
|
||||||
|
<property name="option" value="EOL"/>
|
||||||
|
</module>
|
||||||
|
<module name="SeparatorWrap">
|
||||||
|
<!-- ARRAY_DECLARATOR is EOL until https://github.com/google/styleguide/issues/259 -->
|
||||||
|
<property name="id" value="SeparatorWrapArrayDeclarator"/>
|
||||||
|
<property name="tokens" value="ARRAY_DECLARATOR"/>
|
||||||
|
<property name="option" value="EOL"/>
|
||||||
|
</module>
|
||||||
|
<module name="SeparatorWrap">
|
||||||
|
<property name="id" value="SeparatorWrapMethodRef"/>
|
||||||
|
<property name="tokens" value="METHOD_REF"/>
|
||||||
|
<property name="option" value="nl"/>
|
||||||
|
</module>
|
||||||
|
<module name="PackageName">
|
||||||
|
<property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Package name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="TypeName">
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Type name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="MemberName">
|
||||||
|
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9]*$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Member name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="ParameterName">
|
||||||
|
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="CatchParameterName">
|
||||||
|
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Catch parameter name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="LocalVariableName">
|
||||||
|
<property name="tokens" value="VARIABLE_DEF"/>
|
||||||
|
<property name="format" value="^[a-z]([a-z0-9][a-zA-Z0-9]*)?$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Local variable name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="ClassTypeParameterName">
|
||||||
|
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Class type name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="MethodTypeParameterName">
|
||||||
|
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Method type name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="InterfaceTypeParameterName">
|
||||||
|
<property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Interface type name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<module name="NoFinalizer"/>
|
||||||
|
<module name="GenericWhitespace">
|
||||||
|
<message key="ws.followed"
|
||||||
|
value="GenericWhitespace ''{0}'' is followed by whitespace."/>
|
||||||
|
<message key="ws.preceded"
|
||||||
|
value="GenericWhitespace ''{0}'' is preceded with whitespace."/>
|
||||||
|
<message key="ws.illegalFollow"
|
||||||
|
value="GenericWhitespace ''{0}'' should followed by whitespace."/>
|
||||||
|
<message key="ws.notPreceded"
|
||||||
|
value="GenericWhitespace ''{0}'' is not preceded with whitespace."/>
|
||||||
|
</module>
|
||||||
|
<module name="Indentation">
|
||||||
|
<property name="basicOffset" value="2"/>
|
||||||
|
<property name="braceAdjustment" value="0"/>
|
||||||
|
<property name="caseIndent" value="2"/>
|
||||||
|
<property name="throwsIndent" value="4"/>
|
||||||
|
<property name="lineWrappingIndentation" value="4"/>
|
||||||
|
<property name="arrayInitIndent" value="2"/>
|
||||||
|
</module>
|
||||||
|
<!-- <module name="AbbreviationAsWordInName">
|
||||||
|
<property name="ignoreFinal" value="false"/>
|
||||||
|
<property name="allowedAbbreviationLength" value="4"/>
|
||||||
|
</module> -->
|
||||||
|
<module name="OverloadMethodsDeclarationOrder"/>
|
||||||
|
<module name="VariableDeclarationUsageDistance"/>
|
||||||
|
<module name="CustomImportOrder">
|
||||||
|
<property name="sortImportsInGroupAlphabetically" value="true"/>
|
||||||
|
<property name="separateLineBetweenGroups" value="true"/>
|
||||||
|
<property name="customImportOrderRules" value="STATIC###THIRD_PARTY_PACKAGE"/>
|
||||||
|
</module>
|
||||||
|
<module name="MethodParamPad"/>
|
||||||
|
<module name="NoWhitespaceBefore">
|
||||||
|
<property name="tokens" value="COMMA, SEMI, POST_INC, POST_DEC, DOT, ELLIPSIS, METHOD_REF"/>
|
||||||
|
<property name="allowLineBreaks" value="true"/>
|
||||||
|
</module>
|
||||||
|
<module name="ParenPad"/>
|
||||||
|
<module name="OperatorWrap">
|
||||||
|
<property name="option" value="NL"/>
|
||||||
|
<property name="tokens" value="BAND, BOR, BSR, BXOR, DIV, EQUAL, GE, GT, LAND, LE, LITERAL_INSTANCEOF, LOR, LT, MINUS, MOD, NOT_EQUAL, PLUS, QUESTION, SL, SR, STAR, METHOD_REF "/>
|
||||||
|
</module>
|
||||||
|
<module name="AnnotationLocation">
|
||||||
|
<property name="id" value="AnnotationLocationMostCases"/>
|
||||||
|
<property name="tokens" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF"/>
|
||||||
|
</module>
|
||||||
|
<module name="AnnotationLocation">
|
||||||
|
<property name="id" value="AnnotationLocationVariables"/>
|
||||||
|
<property name="tokens" value="VARIABLE_DEF"/>
|
||||||
|
<property name="allowSamelineMultipleAnnotations" value="true"/>
|
||||||
|
</module>
|
||||||
|
<!-- <module name="NonEmptyAtclauseDescription"/> -->
|
||||||
|
<!-- <module name="JavadocTagContinuationIndentation"/> -->
|
||||||
|
<!-- <module name="SummaryJavadoc">
|
||||||
|
<property name="forbiddenSummaryFragments" value="^@return the *|^This method returns |^A [{]@code [a-zA-Z0-9]+[}]( is a )"/>
|
||||||
|
</module> -->
|
||||||
|
<!-- <module name="JavadocParagraph"/> -->
|
||||||
|
<!-- <module name="AtclauseOrder">
|
||||||
|
<property name="tagOrder" value="@param, @return, @throws, @deprecated"/>
|
||||||
|
<property name="target" value="CLASS_DEF, INTERFACE_DEF, ENUM_DEF, METHOD_DEF, CTOR_DEF, VARIABLE_DEF"/>
|
||||||
|
</module> -->
|
||||||
|
<!-- <module name="JavadocMethod">
|
||||||
|
<property name="scope" value="public"/>
|
||||||
|
<property name="allowMissingParamTags" value="true"/>
|
||||||
|
<property name="allowMissingThrowsTags" value="true"/>
|
||||||
|
<property name="allowMissingReturnTag" value="true"/>
|
||||||
|
<property name="minLineCount" value="2"/>
|
||||||
|
<property name="allowedAnnotations" value="Override, Test"/>
|
||||||
|
<property name="allowThrowsTagsForSubclasses" value="true"/>
|
||||||
|
</module> -->
|
||||||
|
<module name="MethodName">
|
||||||
|
<property name="format" value="^[a-z][a-z0-9][a-zA-Z0-9_]*$"/>
|
||||||
|
<message key="name.invalidPattern"
|
||||||
|
value="Method name ''{0}'' must match pattern ''{1}''."/>
|
||||||
|
</module>
|
||||||
|
<!-- <module name="SingleLineJavadoc">
|
||||||
|
<property name="ignoreInlineTags" value="false"/>
|
||||||
|
</module> -->
|
||||||
|
<module name="EmptyCatchBlock">
|
||||||
|
<property name="exceptionVariableName" value="expected"/>
|
||||||
|
</module>
|
||||||
|
<module name="CommentsIndentation"/>
|
||||||
|
<module name="SuppressionCommentFilter"/>
|
||||||
|
</module>
|
||||||
|
</module>
|
||||||
|
|
@ -0,0 +1,2 @@
|
||||||
|
[pycodestyle]
|
||||||
|
ignore = E501
|
||||||
|
|
@ -0,0 +1,378 @@
|
||||||
|
[MASTER]
|
||||||
|
|
||||||
|
# Specify a configuration file.
|
||||||
|
#rcfile=
|
||||||
|
|
||||||
|
# Python code to execute. Adds the analysis directory to the system path
|
||||||
|
# variable which is necessary to avoid ImportErrors in our website code.
|
||||||
|
init-hook='import os, sys; cwd = os.getcwd(); analysis_path = os.path.join(cwd, 'server') if '/server/' not in cwd else cwd[0:cwd.index('/server/') + len('/server/') - 1]; sys.path.insert(0, analysis_path)'
|
||||||
|
|
||||||
|
# Add files or directories to the blacklist. They should be base names, not
|
||||||
|
# paths.
|
||||||
|
ignore=CVS
|
||||||
|
|
||||||
|
# Pickle collected data for later comparisons.
|
||||||
|
persistent=yes
|
||||||
|
|
||||||
|
# List of plugins (as comma separated values of python modules names) to load,
|
||||||
|
# usually to register additional checkers.
|
||||||
|
load-plugins=
|
||||||
|
|
||||||
|
# Use multiple processes to speed up Pylint.
|
||||||
|
jobs=1
|
||||||
|
|
||||||
|
# Allow loading of arbitrary C extensions. Extensions are imported into the
|
||||||
|
# active Python interpreter and may run arbitrary code.
|
||||||
|
unsafe-load-any-extension=no
|
||||||
|
|
||||||
|
# A comma-separated list of package or module names from where C extensions may
|
||||||
|
# be loaded. Extensions are loading into the active Python interpreter and may
|
||||||
|
# run arbitrary code
|
||||||
|
extension-pkg-whitelist=
|
||||||
|
|
||||||
|
# Allow optimization of some AST trees. This will activate a peephole AST
|
||||||
|
# optimizer, which will apply various small optimizations. For instance, it can
|
||||||
|
# be used to obtain the result of joining multiple strings with the addition
|
||||||
|
# operator. Joining a lot of strings can lead to a maximum recursion error in
|
||||||
|
# Pylint and this flag can prevent that. It has one side effect, the resulting
|
||||||
|
# AST will be different than the one from reality.
|
||||||
|
optimize-ast=no
|
||||||
|
|
||||||
|
|
||||||
|
[MESSAGES CONTROL]
|
||||||
|
|
||||||
|
# Only show warnings with the listed confidence levels. Leave empty to show
|
||||||
|
# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
|
||||||
|
confidence=HIGH,INFERENCE_FAILURE,UNDEFINED
|
||||||
|
|
||||||
|
# Enable the message, report, category or checker with the given id(s). You can
|
||||||
|
# either give multiple identifier separated by comma (,) or put this option
|
||||||
|
# multiple time. See also the "--disable" option for examples.
|
||||||
|
#enable=
|
||||||
|
|
||||||
|
# Disable the message, report, category or checker with the given id(s). You
|
||||||
|
# can either give multiple identifiers separated by comma (,) or put this
|
||||||
|
# option multiple times (only on the command line, not in the configuration
|
||||||
|
# file where it should appear only once).You can also use "--disable=all" to
|
||||||
|
# disable everything first and then reenable specific checks. For example, if
|
||||||
|
# you want to run only the similarities checker, you can use "--disable=all
|
||||||
|
# --enable=similarities". If you want to run only the classes checker, but have
|
||||||
|
# no Warning level messages displayed, use"--disable=all --enable=classes
|
||||||
|
# --disable=W"
|
||||||
|
disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,missing-docstring,too-few-public-methods,too-many-arguments,too-many-locals,too-many-instance-attributes,too-many-statements,locally-disabled,superfluous-parens,too-many-branches,not-callable,too-many-nested-blocks,fixme,redefined-variable-type,no-member,locally-enabled,too-many-public-methods
|
||||||
|
|
||||||
|
|
||||||
|
[REPORTS]
|
||||||
|
|
||||||
|
# Set the output format. Available formats are text, parseable, colorized, msvs
|
||||||
|
# (visual studio) and html. You can also give a reporter class, eg
|
||||||
|
# mypackage.mymodule.MyReporterClass.
|
||||||
|
output-format=text
|
||||||
|
|
||||||
|
# Put messages in a separate file for each module / package specified on the
|
||||||
|
# command line instead of printing them on stdout. Reports (if any) will be
|
||||||
|
# written in a file name "pylint_global.[txt|html]".
|
||||||
|
files-output=no
|
||||||
|
|
||||||
|
# Tells whether to display a full report or only the messages
|
||||||
|
reports=no
|
||||||
|
|
||||||
|
# Python expression which should return a note less than 10 (10 is the highest
|
||||||
|
# note). You have access to the variables errors warning, statement which
|
||||||
|
# respectively contain the number of errors / warnings messages and the total
|
||||||
|
# number of statements analyzed. This is used by the global evaluation report
|
||||||
|
# (RP0004).
|
||||||
|
evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
|
||||||
|
|
||||||
|
# Template used to display messages. This is a python new-style format string
|
||||||
|
# used to format the message information. See doc for all details
|
||||||
|
#msg-template=
|
||||||
|
|
||||||
|
|
||||||
|
[SIMILARITIES]
|
||||||
|
|
||||||
|
# Minimum lines number of a similarity.
|
||||||
|
min-similarity-lines=4
|
||||||
|
|
||||||
|
# Ignore comments when computing similarities.
|
||||||
|
ignore-comments=yes
|
||||||
|
|
||||||
|
# Ignore docstrings when computing similarities.
|
||||||
|
ignore-docstrings=yes
|
||||||
|
|
||||||
|
# Ignore imports when computing similarities.
|
||||||
|
ignore-imports=no
|
||||||
|
|
||||||
|
|
||||||
|
[TYPECHECK]
|
||||||
|
|
||||||
|
# Tells whether missing members accessed in mixin class should be ignored. A
|
||||||
|
# mixin class is detected if its name ends with "mixin" (case insensitive).
|
||||||
|
ignore-mixin-members=yes
|
||||||
|
|
||||||
|
# List of module names for which member attributes should not be checked
|
||||||
|
# (useful for modules/projects where namespaces are manipulated during runtime
|
||||||
|
# and thus existing member attributes cannot be deduced by static analysis. It
|
||||||
|
# supports qualified module names, as well as Unix pattern matching.
|
||||||
|
ignored-modules=
|
||||||
|
|
||||||
|
# List of classes names for which member attributes should not be checked
|
||||||
|
# (useful for classes with attributes dynamically set). This supports can work
|
||||||
|
# with qualified names.
|
||||||
|
ignored-classes=
|
||||||
|
|
||||||
|
# List of members which are set dynamically and missed by pylint inference
|
||||||
|
# system, and so shouldn't trigger E1101 when accessed. Python regular
|
||||||
|
# expressions are accepted.
|
||||||
|
generated-members=
|
||||||
|
|
||||||
|
|
||||||
|
[LOGGING]
|
||||||
|
|
||||||
|
# Logging modules to check that the string format arguments are in logging
|
||||||
|
# function parameter format
|
||||||
|
logging-modules=logging
|
||||||
|
|
||||||
|
|
||||||
|
[SPELLING]
|
||||||
|
|
||||||
|
# Spelling dictionary name. Available dictionaries: none. To make it working
|
||||||
|
# install python-enchant package.
|
||||||
|
spelling-dict=
|
||||||
|
|
||||||
|
# List of comma separated words that should not be checked.
|
||||||
|
spelling-ignore-words=
|
||||||
|
|
||||||
|
# A path to a file that contains private dictionary; one word per line.
|
||||||
|
spelling-private-dict-file=
|
||||||
|
|
||||||
|
# Tells whether to store unknown words to indicated private dictionary in
|
||||||
|
# --spelling-private-dict-file option instead of raising a message.
|
||||||
|
spelling-store-unknown-words=no
|
||||||
|
|
||||||
|
|
||||||
|
[VARIABLES]
|
||||||
|
|
||||||
|
# Tells whether we should check for unused import in __init__ files.
|
||||||
|
init-import=no
|
||||||
|
|
||||||
|
# A regular expression matching the name of dummy variables (i.e. expectedly
|
||||||
|
# not used).
|
||||||
|
dummy-variables-rgx=_$|dummy
|
||||||
|
|
||||||
|
# List of additional names supposed to be defined in builtins. Remember that
|
||||||
|
# you should avoid to define new builtins when possible.
|
||||||
|
additional-builtins=
|
||||||
|
|
||||||
|
# List of strings which can identify a callback function by name. A callback
|
||||||
|
# name must start or end with one of those strings.
|
||||||
|
callbacks=cb_,_cb
|
||||||
|
|
||||||
|
|
||||||
|
[FORMAT]
|
||||||
|
|
||||||
|
# Maximum number of characters on a single line.
|
||||||
|
max-line-length=100
|
||||||
|
|
||||||
|
# Regexp for a line that is allowed to be longer than the limit.
|
||||||
|
ignore-long-lines=^\s*(# )?<?https?://\S+>?$
|
||||||
|
|
||||||
|
# Allow the body of an if to be on the same line as the test if there is no
|
||||||
|
# else.
|
||||||
|
single-line-if-stmt=no
|
||||||
|
|
||||||
|
# List of optional constructs for which whitespace checking is disabled. `dict-
|
||||||
|
# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}.
|
||||||
|
# `trailing-comma` allows a space between comma and closing bracket: (a, ).
|
||||||
|
# `empty-line` allows space-only lines.
|
||||||
|
no-space-check=trailing-comma,dict-separator
|
||||||
|
|
||||||
|
# Maximum number of lines in a module
|
||||||
|
max-module-lines=1000
|
||||||
|
|
||||||
|
# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1
|
||||||
|
# tab).
|
||||||
|
indent-string=' '
|
||||||
|
|
||||||
|
# Number of spaces of indent required inside a hanging or continued line.
|
||||||
|
indent-after-paren=4
|
||||||
|
|
||||||
|
# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
|
||||||
|
expected-line-ending-format=
|
||||||
|
|
||||||
|
|
||||||
|
[MISCELLANEOUS]
|
||||||
|
|
||||||
|
# List of note tags to take in consideration, separated by a comma.
|
||||||
|
notes=FIXME,XXX,TODO
|
||||||
|
|
||||||
|
|
||||||
|
[BASIC]
|
||||||
|
|
||||||
|
# List of builtins function names that should not be used, separated by a comma
|
||||||
|
bad-functions=map,filter,input
|
||||||
|
|
||||||
|
# Good variable names which should always be accepted, separated by a comma
|
||||||
|
good-names=ex,Run,_,mu,y
|
||||||
|
|
||||||
|
# Bad variable names which should always be refused, separated by a comma
|
||||||
|
bad-names=foo,bar,baz,toto,tutu,tata
|
||||||
|
|
||||||
|
# Colon-delimited sets of names that determine each other's naming style when
|
||||||
|
# the name regexes allow several styles.
|
||||||
|
name-group=
|
||||||
|
|
||||||
|
# Include a hint for the correct naming format with invalid-name
|
||||||
|
include-naming-hint=no
|
||||||
|
|
||||||
|
# Regular expression matching correct function names
|
||||||
|
function-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||||
|
|
||||||
|
# Naming hint for function names
|
||||||
|
function-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||||
|
|
||||||
|
# Regular expression matching correct variable names
|
||||||
|
variable-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$
|
||||||
|
|
||||||
|
# Naming hint for variable names
|
||||||
|
variable-name-hint=[a-z_][a-z0-9_]{0,30}$
|
||||||
|
|
||||||
|
# Regular expression matching correct constant names
|
||||||
|
const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||||
|
|
||||||
|
# Naming hint for constant names
|
||||||
|
const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$
|
||||||
|
|
||||||
|
# Regular expression matching correct attribute names
|
||||||
|
attr-rgx=(([a-z_][a-z0-9_]{1,30})|([XK][a-z0-9_]{0,30}))$
|
||||||
|
|
||||||
|
# Naming hint for attribute names
|
||||||
|
attr-name-hint=[a-z_][a-z0-9_]{1,30}$
|
||||||
|
|
||||||
|
# Regular expression matching correct argument names
|
||||||
|
argument-rgx=(([a-z_][a-z0-9_]{0,30})|([XK][a-z0-9_]{0,30}))$
|
||||||
|
|
||||||
|
# Naming hint for argument names
|
||||||
|
argument-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||||
|
|
||||||
|
# Regular expression matching correct class attribute names
|
||||||
|
class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||||
|
|
||||||
|
# Naming hint for class attribute names
|
||||||
|
class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$
|
||||||
|
|
||||||
|
# Regular expression matching correct inline iteration names
|
||||||
|
inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$
|
||||||
|
|
||||||
|
# Naming hint for inline iteration names
|
||||||
|
inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$
|
||||||
|
|
||||||
|
# Regular expression matching correct class names
|
||||||
|
class-rgx=[A-Z_][a-zA-Z0-9]+$
|
||||||
|
|
||||||
|
# Naming hint for class names
|
||||||
|
class-name-hint=[A-Z_][a-zA-Z0-9]+$
|
||||||
|
|
||||||
|
# Regular expression matching correct module names
|
||||||
|
module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||||
|
|
||||||
|
# Naming hint for module names
|
||||||
|
module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$
|
||||||
|
|
||||||
|
# Regular expression matching correct method names
|
||||||
|
method-rgx=[a-z_][a-z0-9_]{2,30}$
|
||||||
|
|
||||||
|
# Naming hint for method names
|
||||||
|
method-name-hint=[a-z_][a-z0-9_]{2,30}$
|
||||||
|
|
||||||
|
# Regular expression which should only match function or class names that do
|
||||||
|
# not require a docstring.
|
||||||
|
no-docstring-rgx=^_
|
||||||
|
|
||||||
|
# Minimum line length for functions/classes that require docstrings, shorter
|
||||||
|
# ones are exempt.
|
||||||
|
docstring-min-length=-1
|
||||||
|
|
||||||
|
|
||||||
|
[ELIF]
|
||||||
|
|
||||||
|
# Maximum number of nested blocks for function / method body
|
||||||
|
max-nested-blocks=5
|
||||||
|
|
||||||
|
|
||||||
|
[CLASSES]
|
||||||
|
|
||||||
|
# List of method names used to declare (i.e. assign) instance attributes.
|
||||||
|
defining-attr-methods=__init__,__new__,setUp
|
||||||
|
|
||||||
|
# List of valid names for the first argument in a class method.
|
||||||
|
valid-classmethod-first-arg=cls
|
||||||
|
|
||||||
|
# List of valid names for the first argument in a metaclass class method.
|
||||||
|
valid-metaclass-classmethod-first-arg=mcs
|
||||||
|
|
||||||
|
# List of member names, which should be excluded from the protected access
|
||||||
|
# warning.
|
||||||
|
exclude-protected=_asdict,_fields,_replace,_source,_make
|
||||||
|
|
||||||
|
|
||||||
|
[IMPORTS]
|
||||||
|
|
||||||
|
# Deprecated modules which should not be used, separated by a comma
|
||||||
|
deprecated-modules=regsub,TERMIOS,Bastion,rexec
|
||||||
|
|
||||||
|
# Create a graph of every (i.e. internal and external) dependencies in the
|
||||||
|
# given file (report RP0402 must not be disabled)
|
||||||
|
import-graph=
|
||||||
|
|
||||||
|
# Create a graph of external dependencies in the given file (report RP0402 must
|
||||||
|
# not be disabled)
|
||||||
|
ext-import-graph=
|
||||||
|
|
||||||
|
# Create a graph of internal dependencies in the given file (report RP0402 must
|
||||||
|
# not be disabled)
|
||||||
|
int-import-graph=
|
||||||
|
|
||||||
|
|
||||||
|
[DESIGN]
|
||||||
|
|
||||||
|
# Maximum number of arguments for function / method
|
||||||
|
max-args=5
|
||||||
|
|
||||||
|
# Argument names that match this expression will be ignored. Default to name
|
||||||
|
# with leading underscore
|
||||||
|
ignored-argument-names=_.*
|
||||||
|
|
||||||
|
# Maximum number of locals for function / method body
|
||||||
|
max-locals=15
|
||||||
|
|
||||||
|
# Maximum number of return / yield for function / method body
|
||||||
|
max-returns=6
|
||||||
|
|
||||||
|
# Maximum number of branch for function / method body
|
||||||
|
max-branches=12
|
||||||
|
|
||||||
|
# Maximum number of statements in function / method body
|
||||||
|
max-statements=50
|
||||||
|
|
||||||
|
# Maximum number of parents for a class (see R0901).
|
||||||
|
max-parents=7
|
||||||
|
|
||||||
|
# Maximum number of attributes for a class (see R0902).
|
||||||
|
max-attributes=7
|
||||||
|
|
||||||
|
# Minimum number of public methods for a class (see R0903).
|
||||||
|
min-public-methods=2
|
||||||
|
|
||||||
|
# Maximum number of public methods for a class (see R0904).
|
||||||
|
max-public-methods=20
|
||||||
|
|
||||||
|
# Maximum number of boolean expressions in a if statement
|
||||||
|
max-bool-expr=5
|
||||||
|
|
||||||
|
|
||||||
|
[EXCEPTIONS]
|
||||||
|
|
||||||
|
# Exceptions that will emit a warning when being caught. Defaults to
|
||||||
|
# "Exception"
|
||||||
|
overgeneral-exceptions=Exception
|
||||||
|
|
@ -0,0 +1,198 @@
|
||||||
|
#
|
||||||
|
# OtterTune - formatter.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
import argparse
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import autopep8
|
||||||
|
|
||||||
|
EXIT_SUCCESS = 0
|
||||||
|
EXIT_FAILURE = -1
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# LOGGING CONFIGURATION
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
LOG_HANDLER = logging.StreamHandler()
|
||||||
|
LOG_FORMATTER = logging.Formatter(
|
||||||
|
fmt='%(asctime)s [%(funcName)s:%(lineno)03d] %(levelname)-5s: %(message)s',
|
||||||
|
datefmt='%H:%M:%S'
|
||||||
|
)
|
||||||
|
LOG_HANDLER.setFormatter(LOG_FORMATTER)
|
||||||
|
LOG.addHandler(LOG_HANDLER)
|
||||||
|
LOG.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# CONFIGURATION
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
# NOTE: the absolute path to ottertune directory is calculated from current
|
||||||
|
# directory structure: ottertune/server/website/scripts/validators/<this_file>
|
||||||
|
# OTTERTUNE_DIR needs to be redefined if the directory structure is changed.
|
||||||
|
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
OTTERTUNE_DIR = os.path.abspath(functools.reduce(os.path.join,
|
||||||
|
[CODE_SOURCE_DIR,
|
||||||
|
os.path.pardir,
|
||||||
|
os.path.pardir]))
|
||||||
|
|
||||||
|
JAVA_JAR_PATH = os.path.join(
|
||||||
|
OTTERTUNE_DIR, 'controller/build/libs/google-java-format-1.5-all-deps.jar')
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# FILE HEADER FORMATS
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
PYTHON_HEADER_FORMAT = (
|
||||||
|
"#\n"
|
||||||
|
"# OtterTune - {filename}\n"
|
||||||
|
"#\n"
|
||||||
|
"# Copyright (c) 2017-18, Carnegie Mellon University Database Group\n"
|
||||||
|
"#\n"
|
||||||
|
).format
|
||||||
|
|
||||||
|
# Regex for updating old headers
|
||||||
|
PYTHON_HEADER_REGEX = re.compile(r'#\n#.*\n#\n# Copyright.*\n#\n')
|
||||||
|
|
||||||
|
JAVA_HEADER_FORMAT = (
|
||||||
|
"/*\n"
|
||||||
|
" * OtterTune - {filename}\n"
|
||||||
|
" *\n"
|
||||||
|
" * Copyright (c) 2017-18, Carnegie Mellon University Database Group\n"
|
||||||
|
" */\n\n"
|
||||||
|
).format
|
||||||
|
|
||||||
|
JAVA_HEADER_REGEX = re.compile(r'/\*\n \*.*\n \*\n \* Copyright.*\n \*/\n\n')
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# UTILITY FUNCTION DEFINITIONS
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
|
||||||
|
def format_file(file_path, update_header, format_code):
|
||||||
|
if file_path.endswith(".py"):
|
||||||
|
format_python_file(file_path, update_header, format_code)
|
||||||
|
elif file_path.endswith(".java"):
|
||||||
|
format_java_file(file_path, update_header, format_code)
|
||||||
|
|
||||||
|
|
||||||
|
def update_file_header(file_contents, file_name, header_format, header_regex):
|
||||||
|
new_header = header_format(filename=os.path.basename(file_name))
|
||||||
|
header_match = header_regex.search(file_contents)
|
||||||
|
if header_match:
|
||||||
|
# Replace the old header with the new one
|
||||||
|
old_header = header_match.group()
|
||||||
|
file_contents = file_contents.replace(old_header, new_header)
|
||||||
|
else:
|
||||||
|
# Add new header
|
||||||
|
file_contents = new_header + file_contents
|
||||||
|
return file_contents
|
||||||
|
|
||||||
|
|
||||||
|
def format_java_file(file_path, update_header, format_code):
|
||||||
|
if not file_path.endswith(".java"):
|
||||||
|
return
|
||||||
|
|
||||||
|
if update_header:
|
||||||
|
with open(file_path, 'r') as f:
|
||||||
|
file_contents = f.read()
|
||||||
|
file_contents = update_file_header(file_contents,
|
||||||
|
os.path.basename(file_path),
|
||||||
|
JAVA_HEADER_FORMAT,
|
||||||
|
JAVA_HEADER_REGEX)
|
||||||
|
with open(file_path, 'w') as f:
|
||||||
|
f.write(file_contents)
|
||||||
|
|
||||||
|
if format_code:
|
||||||
|
if not os.path.exists(JAVA_JAR_PATH):
|
||||||
|
controller_dir = os.path.join(OTTERTUNE_DIR, 'controller')
|
||||||
|
subprocess.check_output(["gradle", "downloadJars"], cwd=controller_dir)
|
||||||
|
subprocess.check_output(["java", "-jar", JAVA_JAR_PATH, "-r", file_path])
|
||||||
|
|
||||||
|
|
||||||
|
def format_python_file(file_path, update_header, format_code):
|
||||||
|
if not file_path.endswith(".py"):
|
||||||
|
return
|
||||||
|
|
||||||
|
with open(file_path, 'r') as f:
|
||||||
|
file_contents = f.read()
|
||||||
|
|
||||||
|
if update_header:
|
||||||
|
file_contents = update_file_header(file_contents,
|
||||||
|
os.path.basename(file_path),
|
||||||
|
PYTHON_HEADER_FORMAT,
|
||||||
|
PYTHON_HEADER_REGEX)
|
||||||
|
|
||||||
|
if format_code:
|
||||||
|
# Use the autopep8 module to format the source code. autopep8 uses
|
||||||
|
# pycodestyle to detect the style errors it should fix and thus it
|
||||||
|
# should fix all (or most) of them, however, it does not use pylint
|
||||||
|
# so it may not fix all of its reported errors.
|
||||||
|
options = {"max_line_length": 100}
|
||||||
|
file_contents = autopep8.fix_code(file_contents, options=options)
|
||||||
|
|
||||||
|
with open(file_path, 'w') as f:
|
||||||
|
f.write(file_contents)
|
||||||
|
|
||||||
|
|
||||||
|
# Format all the files in the dir passed as argument
|
||||||
|
def format_dir(dir_path, update_header, format_code):
|
||||||
|
for subdir, _, files in os.walk(dir_path): # pylint: disable=not-an-iterable
|
||||||
|
for file_path in files:
|
||||||
|
file_path = subdir + os.path.sep + file_path
|
||||||
|
format_file(file_path, update_header, format_code)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description='Formats python source files in place')
|
||||||
|
parser.add_argument('--no-update-header', action='store_true',
|
||||||
|
help='Do not update the source file headers')
|
||||||
|
parser.add_argument('--no-format-code', action='store_true',
|
||||||
|
help='Do not format the source files use autopep8')
|
||||||
|
parser.add_argument('--staged-files', action='store_true',
|
||||||
|
help='Apply the selected action(s) to all staged files (git)')
|
||||||
|
parser.add_argument('paths', metavar='PATH', type=str, nargs='*',
|
||||||
|
help='Files or directories to (recursively) apply the actions to')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.no_update_header and args.no_format_code:
|
||||||
|
LOG.info("No actions to perform (both --no-update-header and "
|
||||||
|
"--no-format-code given). Exiting...")
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
elif args.staged_files:
|
||||||
|
targets = [os.path.abspath(os.path.join(OTTERTUNE_DIR, f))
|
||||||
|
for f in subprocess.check_output(["git", "diff",
|
||||||
|
"--name-only", "HEAD",
|
||||||
|
"--cached",
|
||||||
|
"--diff-filter=d"]).split()]
|
||||||
|
if not targets:
|
||||||
|
LOG.error("No staged files or not calling from a repository. Exiting...")
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
elif not args.paths:
|
||||||
|
LOG.error("No files or directories given. Exiting...")
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
else:
|
||||||
|
targets = args.paths
|
||||||
|
|
||||||
|
for x in targets:
|
||||||
|
if os.path.isfile(x):
|
||||||
|
LOG.info("Scanning file: " + x)
|
||||||
|
format_file(x, not args.no_update_header, not args.no_format_code)
|
||||||
|
elif os.path.isdir(x):
|
||||||
|
LOG.info("Scanning directory: " + x)
|
||||||
|
format_dir(x, not args.no_update_header, not args.no_format_code)
|
||||||
|
else:
|
||||||
|
LOG.error("%s isn't a file or directory", x)
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,79 @@
|
||||||
|
#!/bin/sh
|
||||||
|
# Source validation pre-commit hook
|
||||||
|
#
|
||||||
|
# Adapted from the source validation pre-commit hook used in Peloton.
|
||||||
|
# (see https://github.com/cmu-db/peloton/blob/master/script/git-hooks/pre-commit)
|
||||||
|
#
|
||||||
|
# This script collects all modified files and runs it through our source code
|
||||||
|
# validation script. The validation script returns 0 on success and 1 on any
|
||||||
|
# failure. This script can also run the server and controller tests by
|
||||||
|
# uncommenting lines 26-28 and 31-33, respectively.
|
||||||
|
#
|
||||||
|
# To enable, symlink this file to '.git/hooks/pre-commit' like so:
|
||||||
|
# cd $OTTERTUNE_DIR/.git/hooks
|
||||||
|
# ln -s ../../script/git-hooks/pre-commit ./pre-commit
|
||||||
|
|
||||||
|
|
||||||
|
FILES=$(git diff --name-only HEAD --cached --diff-filter=d | grep '\.\(py\)$')
|
||||||
|
|
||||||
|
SERVER_TESTS_RESULT=0
|
||||||
|
CONTROLLER_TESTS_RESULT=0
|
||||||
|
VALIDATOR_RESULT=0
|
||||||
|
|
||||||
|
if [ -n "$FILES" ]; then
|
||||||
|
|
||||||
|
# Uncomment to run the server tests
|
||||||
|
# cd server/website && python manage.py test -v 2
|
||||||
|
# SERVER_TESTS_RESULT=$?
|
||||||
|
# cd ../..
|
||||||
|
|
||||||
|
# Uncomment to run the controller tests
|
||||||
|
# cd controller && gradle build -q
|
||||||
|
# CONTROLLER_TESTS_RESULT=$?
|
||||||
|
# cd ..
|
||||||
|
|
||||||
|
# Run source code validator
|
||||||
|
python script/validators/source_validator.py $FILES
|
||||||
|
VALIDATOR_RESULT=$?
|
||||||
|
|
||||||
|
if [ "$VALIDATOR_RESULT" -ne 0 ] || [ "$SERVER_TESTS_RESULT" -ne 0 ] || [ "$CONTROLLER_TESTS_RESULT" -ne 0 ]; then
|
||||||
|
|
||||||
|
echo " +------------------------------------------------------------+"
|
||||||
|
echo " | |"
|
||||||
|
echo " | OTTERTUNE PRE-COMMIT HOOK |"
|
||||||
|
echo " | |"
|
||||||
|
echo " +------------------------------------------------------------+"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [ "$SERVER_TESTS_RESULT" -ne 0 ]; then
|
||||||
|
echo " FAILED server tests!"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$CONTROLLER_TESTS_RESULT" -ne 0 ]; then
|
||||||
|
echo " FAILED controller tests!"
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$VALIDATOR_RESULT" -ne 0 ]; then
|
||||||
|
echo " FAILED source validation!"
|
||||||
|
echo ""
|
||||||
|
echo " Use the formatting script to help format all changed files:"
|
||||||
|
echo " (ottertune/script/formatting/formatter.py)"
|
||||||
|
echo ""
|
||||||
|
echo " \"python formatter.py --staged-files\""
|
||||||
|
echo ""
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo " To temporarily bypass the pre-commit hook, use:"
|
||||||
|
echo ""
|
||||||
|
echo " \"git commit --no-verify\""
|
||||||
|
echo
|
||||||
|
echo " Be aware that changed files have to be staged again!"
|
||||||
|
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
|
||||||
|
exit 0
|
||||||
|
|
@ -0,0 +1,54 @@
|
||||||
|
#
|
||||||
|
# OtterTune - query_and_get.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Feb 11, 2018
|
||||||
|
|
||||||
|
@author: taodai
|
||||||
|
'''
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import logging
|
||||||
|
import json
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
LOG.addHandler(logging.StreamHandler())
|
||||||
|
LOG.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
|
||||||
|
# take 3 arguments, save result to next_config in working directory
|
||||||
|
# base_url: for instance, https://0.0.0.0:8000/
|
||||||
|
# upload_code: upload code...
|
||||||
|
# query_interval: time (in second) between queries
|
||||||
|
def main():
|
||||||
|
base_url = sys.argv[1].strip('/')
|
||||||
|
upload_code = sys.argv[2]
|
||||||
|
query_interval = int(sys.argv[3])
|
||||||
|
request = base_url + '/query_and_get/' + upload_code
|
||||||
|
timer = 0
|
||||||
|
start = time.time()
|
||||||
|
while True:
|
||||||
|
response = urllib.request.urlopen(request).read().decode()
|
||||||
|
if 'Fail' in response:
|
||||||
|
LOG.info('Tuning failed\n')
|
||||||
|
break
|
||||||
|
elif response == 'null' or 'not ready' in response:
|
||||||
|
time.sleep(query_interval)
|
||||||
|
timer += query_interval
|
||||||
|
LOG.info('%s s\n', str(timer))
|
||||||
|
else:
|
||||||
|
next_conf_f = open('next_config', 'w')
|
||||||
|
next_conf_f.write(json.loads(response))
|
||||||
|
next_conf_f.close()
|
||||||
|
break
|
||||||
|
elapsed_time = time.time() - start
|
||||||
|
LOG.info('Elapsed time: %s\n', str(elapsed_time))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,437 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# encoding: utf-8
|
||||||
|
#
|
||||||
|
# OtterTune - source_validator.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# SOURCE VALIDATOR
|
||||||
|
# ==============================================
|
||||||
|
#
|
||||||
|
# Adapted from the source validator used by Peloton.
|
||||||
|
# (see https://github.com/cmu-db/peloton/blob/master/script/validators/source_validator.py)
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
import imp
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
import functools
|
||||||
|
from collections import namedtuple
|
||||||
|
from fabric.api import lcd, local, settings, quiet
|
||||||
|
|
||||||
|
EXIT_SUCCESS = 0
|
||||||
|
EXIT_FAILURE = -1
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# CONFIGURATION
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
# Logging
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
LOG.addHandler(logging.StreamHandler())
|
||||||
|
LOG.setLevel(logging.INFO)
|
||||||
|
|
||||||
|
# NOTE: the absolute path to ottertune directory is calculated from current
|
||||||
|
# directory structure: ottertune/server/website/scripts/validators/<this_file>
|
||||||
|
# OTTERTUNE_DIR needs to be redefined if the directory structure is changed.
|
||||||
|
CODE_SOURCE_DIR = os.path.abspath(os.path.dirname(__file__))
|
||||||
|
OTTERTUNE_DIR = os.path.abspath(functools.reduce(os.path.join,
|
||||||
|
[CODE_SOURCE_DIR,
|
||||||
|
os.path.pardir,
|
||||||
|
os.path.pardir]))
|
||||||
|
|
||||||
|
# Other directory paths used are relative to OTTERTUNE_DIR
|
||||||
|
DEFAULT_DIRS = [
|
||||||
|
OTTERTUNE_DIR
|
||||||
|
]
|
||||||
|
|
||||||
|
# Directories that should NOT be checked
|
||||||
|
EXCLUDE_DIRECTORIES = [
|
||||||
|
# Django-generated directories
|
||||||
|
os.path.join(OTTERTUNE_DIR, "server/website/website/migrations"),
|
||||||
|
|
||||||
|
# Source code files from json.org
|
||||||
|
os.path.join(OTTERTUNE_DIR, "client/controller/src/main/java/com/controller/util/json"),
|
||||||
|
|
||||||
|
# Django settings
|
||||||
|
os.path.join(OTTERTUNE_DIR, 'server/website/website/settings'),
|
||||||
|
|
||||||
|
# Docker files
|
||||||
|
os.path.join(OTTERTUNE_DIR, 'docker'),
|
||||||
|
]
|
||||||
|
|
||||||
|
# Files that should NOT be checked
|
||||||
|
EXCLUDE_FILES = [
|
||||||
|
# Django-generated files
|
||||||
|
os.path.join(OTTERTUNE_DIR, 'server/website/manage.py'),
|
||||||
|
|
||||||
|
# Docker files
|
||||||
|
os.path.join(OTTERTUNE_DIR, 'server/website/createadmin.py'),
|
||||||
|
]
|
||||||
|
|
||||||
|
CHECKSTYLE_JAR_PATH = os.path.join(OTTERTUNE_DIR,
|
||||||
|
"client/controller/build/libs/checkstyle-8.8-all.jar")
|
||||||
|
|
||||||
|
# Regex patterns
|
||||||
|
PYCODESTYLE_COMMENT_PATTERN = re.compile(r'#\s*pycodestyle:\s*disable\s*=\s*[\w\,\s]+$')
|
||||||
|
|
||||||
|
PYTHON_ILLEGAL_PATTERNS = [
|
||||||
|
(re.compile(r'^print[ (]'), "Do not use 'print'. Use the logging module instead.")
|
||||||
|
]
|
||||||
|
|
||||||
|
JAVA_ILLEGAL_PATTERNS = [
|
||||||
|
(re.compile(r'^System.out.println'), "Do not use println. Use the logging module instead.")
|
||||||
|
]
|
||||||
|
|
||||||
|
PYTHON_HEADER_PATTERN = re.compile(r'#\n#.*\n#\n# Copyright.*\n#\n')
|
||||||
|
JAVA_HEADER_PATTERN = re.compile(r'/\*\n \*.*\n \*\n \* Copyright.*\n \*/\n\n')
|
||||||
|
|
||||||
|
# Stdout format strings
|
||||||
|
SEPARATOR = 80 * '-'
|
||||||
|
OUTPUT_FMT = (
|
||||||
|
'' + SEPARATOR + '\n\n'
|
||||||
|
'\033[1m' # start bold text
|
||||||
|
'%s\n'
|
||||||
|
'FAILED: %s\n\n'
|
||||||
|
'\033[0m' # end bold text
|
||||||
|
'%s'
|
||||||
|
)
|
||||||
|
VALIDATOR_FMT = '{name}\n{u}\n{out}'.format
|
||||||
|
MSG_PREFIX_FMT = ' {filename}:{line:3d}: '.format
|
||||||
|
MSG_SUFFIX_FMT = ' ({symbol})'.format
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# UTILITY FUNCTION DEFINITIONS
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
def format_message(filename, line, message, symbol=None):
|
||||||
|
out_prefix = MSG_PREFIX_FMT(filename=filename, line=line)
|
||||||
|
out_suffix = '' if symbol is None else MSG_SUFFIX_FMT(symbol=symbol)
|
||||||
|
|
||||||
|
# Crop the message details to make the output more readable
|
||||||
|
max_msg_len = 80 - len(out_prefix) - len(out_suffix)
|
||||||
|
if len(message) > max_msg_len:
|
||||||
|
message = message[:max_msg_len - 3] + '...'
|
||||||
|
output = (out_prefix + message + out_suffix).replace('\n', '')
|
||||||
|
return output + '\n'
|
||||||
|
|
||||||
|
|
||||||
|
def validate_validator(modules, config_path):
|
||||||
|
status = True
|
||||||
|
|
||||||
|
# Check if required modules are installed
|
||||||
|
for module in modules:
|
||||||
|
if module is not None:
|
||||||
|
try:
|
||||||
|
imp.find_module(module)
|
||||||
|
except ImportError:
|
||||||
|
LOG.error("Cannot find module %s", module)
|
||||||
|
status = False
|
||||||
|
|
||||||
|
# Check that the config file exists if assigned
|
||||||
|
if config_path is not None and not os.path.isfile(config_path):
|
||||||
|
LOG.error("Cannot find config file %s", config_path)
|
||||||
|
status = False
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
# Validate the file passed as argument
|
||||||
|
def validate_file(file_path):
|
||||||
|
if file_path in EXCLUDE_FILES:
|
||||||
|
return True
|
||||||
|
if not file_path.endswith(".py") and not file_path.endswith(".java"):
|
||||||
|
return True
|
||||||
|
|
||||||
|
LOG.debug("Validating file: %s", file_path)
|
||||||
|
status = True
|
||||||
|
output = []
|
||||||
|
failed_validators = []
|
||||||
|
for validator in VALIDATORS:
|
||||||
|
val_status, val_output = validator.validate_fn(
|
||||||
|
file_path, validator.config_path)
|
||||||
|
if not val_status:
|
||||||
|
status = False
|
||||||
|
output.append(VALIDATOR_FMT(name=validator.name,
|
||||||
|
u='-' * len(validator.name),
|
||||||
|
out=val_output))
|
||||||
|
failed_validators.append(validator.name)
|
||||||
|
if not status:
|
||||||
|
LOG.info(OUTPUT_FMT, file_path, ', '.join(failed_validators), '\n'.join(output))
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
# Validate all the files in the root_dir passed as argument
|
||||||
|
def validate_dir(root_dir):
|
||||||
|
if root_dir in EXCLUDE_DIRECTORIES:
|
||||||
|
return True
|
||||||
|
|
||||||
|
status = True
|
||||||
|
for root, dirs, files in os.walk(root_dir): # pylint: disable=not-an-iterable
|
||||||
|
# Remove excluded dirs from list
|
||||||
|
dirs[:] = [d for d in dirs if os.path.join(root, d) not in EXCLUDE_DIRECTORIES]
|
||||||
|
for file_path in files:
|
||||||
|
file_path = os.path.join(root, file_path)
|
||||||
|
|
||||||
|
if not validate_file(file_path):
|
||||||
|
status = False
|
||||||
|
return status
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# VALIDATOR FUNCTION DEFINITIONS
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
def check_pylint(file_path, config_path=None):
|
||||||
|
if not file_path.endswith(".py"):
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
options = [
|
||||||
|
'--output-format=json',
|
||||||
|
'--reports=yes',
|
||||||
|
]
|
||||||
|
if config_path is not None:
|
||||||
|
options.append('--rcfile=' + config_path)
|
||||||
|
|
||||||
|
with settings(warn_only=True), quiet():
|
||||||
|
res = local('pylint {} {}'.format(' '.join(options), file_path), capture=True)
|
||||||
|
if res.stdout == '':
|
||||||
|
assert res.return_code == 0, 'return_code={}, expected=0\n{}'.format(
|
||||||
|
res.return_code, res.stderr)
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
output = []
|
||||||
|
errors = json.loads(res.stdout)
|
||||||
|
for entry in errors:
|
||||||
|
# Remove extra whitespace and hints
|
||||||
|
msg = entry['message'].replace('^', '').replace('|', '')
|
||||||
|
msg = re.sub(' +', ' ', msg)
|
||||||
|
msg = msg.strip()
|
||||||
|
output.append(format_message(os.path.basename(file_path), entry['line'],
|
||||||
|
msg, entry['symbol']))
|
||||||
|
output = ''.join(output)
|
||||||
|
return res.return_code == 0, output
|
||||||
|
|
||||||
|
|
||||||
|
def check_pycodestyle(file_path, config_path=None):
|
||||||
|
import pycodestyle
|
||||||
|
|
||||||
|
if not file_path.endswith(".py"):
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
# A custom reporter class for pycodestyle that checks for disabled errors
|
||||||
|
# and formats the style report output.
|
||||||
|
class CustomReporter(pycodestyle.StandardReport):
|
||||||
|
def get_file_results(self):
|
||||||
|
# Iterates through the lines of code that generated lint errors and
|
||||||
|
# checks if the given error has been disabled for that line via an
|
||||||
|
# inline comment (e.g., # pycodestyle: disable=E201,E226). Those
|
||||||
|
# that have been disabled are not treated as errors.
|
||||||
|
self._deferred_print.sort()
|
||||||
|
results = []
|
||||||
|
prev_line_num = -1
|
||||||
|
prev_line_errs = []
|
||||||
|
for line_number, _, code, text, _ in self._deferred_print:
|
||||||
|
if prev_line_num == line_number:
|
||||||
|
err_codes = prev_line_errs
|
||||||
|
else:
|
||||||
|
line = self.lines[line_number - 1]
|
||||||
|
m = PYCODESTYLE_COMMENT_PATTERN.search(line)
|
||||||
|
if m and m.group(0):
|
||||||
|
err_codes = [ec.strip() for ec in m.group(0).split('=')[1].split(',')]
|
||||||
|
else:
|
||||||
|
err_codes = []
|
||||||
|
prev_line_num = line_number
|
||||||
|
prev_line_errs = err_codes
|
||||||
|
if code in err_codes:
|
||||||
|
# Error is disabled in source
|
||||||
|
continue
|
||||||
|
|
||||||
|
results.append(format_message(os.path.basename(file_path),
|
||||||
|
self.line_offset + line_number,
|
||||||
|
text, code))
|
||||||
|
return results, len(results) == 0
|
||||||
|
# END CustomReporter class
|
||||||
|
|
||||||
|
options = {} if config_path is None else {'config_file': config_path}
|
||||||
|
style = pycodestyle.StyleGuide(quiet=True, **options)
|
||||||
|
|
||||||
|
# Set the reporter option to our custom one
|
||||||
|
style.options.reporter = CustomReporter
|
||||||
|
style.init_report()
|
||||||
|
report = style.check_files([file_path])
|
||||||
|
results, status = report.get_file_results()
|
||||||
|
output = None if status else ''.join(results)
|
||||||
|
return status, output
|
||||||
|
|
||||||
|
|
||||||
|
def check_java_checkstyle(file_path, config_path=None):
|
||||||
|
if not file_path.endswith(".java"):
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
if not os.path.exists(CHECKSTYLE_JAR_PATH):
|
||||||
|
with lcd(os.path.join(OTTERTUNE_DIR, "client/controller")): # pylint: disable=not-context-manager
|
||||||
|
local("gradle downloadJars")
|
||||||
|
|
||||||
|
options = '' if config_path is None else '-c ' + config_path
|
||||||
|
with quiet():
|
||||||
|
res = local("java -jar {} {} {}".format(CHECKSTYLE_JAR_PATH, options, file_path),
|
||||||
|
capture=True)
|
||||||
|
lines = res.stdout.split('\n')
|
||||||
|
assert len(lines) >= 2 and lines[0] == "Starting audit..." and lines[-1] == "Audit done."
|
||||||
|
if len(lines) == 2:
|
||||||
|
return True, None
|
||||||
|
output = []
|
||||||
|
for line in lines[1:-1]:
|
||||||
|
parts = line.strip().split(':')
|
||||||
|
line_number = int(parts[1])
|
||||||
|
text, code = parts[-1].rsplit('[', 1)
|
||||||
|
text = text.strip()
|
||||||
|
code = code[:-1]
|
||||||
|
output.append(format_message(os.path.basename(file_path), line_number, text, code))
|
||||||
|
output = ''.join(output)
|
||||||
|
return False, output
|
||||||
|
|
||||||
|
|
||||||
|
def check_illegal_patterns(file_path, config_path=None): # pylint: disable=unused-argument
|
||||||
|
if file_path.endswith(".py"):
|
||||||
|
illegal_patterns = PYTHON_ILLEGAL_PATTERNS
|
||||||
|
comment = "#"
|
||||||
|
elif file_path.endswith(".java"):
|
||||||
|
illegal_patterns = JAVA_ILLEGAL_PATTERNS
|
||||||
|
comment = "//"
|
||||||
|
else:
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
line_num = 1
|
||||||
|
output = []
|
||||||
|
status = True
|
||||||
|
with open(file_path, 'r') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
for pattern_info in illegal_patterns:
|
||||||
|
if not line.startswith(comment) and pattern_info[0].search(line):
|
||||||
|
output.append(format_message(filename=os.path.basename(file_path),
|
||||||
|
line=line_num,
|
||||||
|
message=pattern_info[1]))
|
||||||
|
status = False
|
||||||
|
line_num += 1
|
||||||
|
output = None if status else ''.join(output)
|
||||||
|
return status, output
|
||||||
|
|
||||||
|
|
||||||
|
def check_header(file_path, config_file=None): # pylint: disable=unused-argument
|
||||||
|
if file_path.endswith(".py"):
|
||||||
|
header_pattern = PYTHON_HEADER_PATTERN
|
||||||
|
elif file_path.endswith(".java"):
|
||||||
|
header_pattern = JAVA_HEADER_PATTERN
|
||||||
|
else:
|
||||||
|
return True, None
|
||||||
|
|
||||||
|
status = True
|
||||||
|
output = None
|
||||||
|
with open(file_path, 'r') as f:
|
||||||
|
file_contents = f.read()
|
||||||
|
|
||||||
|
header_match = header_pattern.search(file_contents)
|
||||||
|
filename = os.path.basename(file_path)
|
||||||
|
if header_match:
|
||||||
|
if filename not in header_match.group(0):
|
||||||
|
status = False
|
||||||
|
output = format_message(filename=filename, line=2,
|
||||||
|
message="Incorrect filename in header")
|
||||||
|
|
||||||
|
else:
|
||||||
|
status = False
|
||||||
|
output = format_message(filename=filename, line=1,
|
||||||
|
message='Missing header')
|
||||||
|
return status, output
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# VALIDATORS
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
# Struct for storing validator metadata
|
||||||
|
Validator = namedtuple('Validator', 'name validate_fn modules config_path')
|
||||||
|
|
||||||
|
VALIDATORS = [
|
||||||
|
# Runs pylint on python source
|
||||||
|
Validator('check_pylint', check_pylint, ['pylint'],
|
||||||
|
os.path.join(OTTERTUNE_DIR, "script/formatting/config/pylintrc")),
|
||||||
|
|
||||||
|
# Runs pycodestyle on python source
|
||||||
|
Validator('check_pycodestyle', check_pycodestyle, ['pycodestyle'],
|
||||||
|
os.path.join(OTTERTUNE_DIR, "script/formatting/config/pycodestyle")),
|
||||||
|
|
||||||
|
# Runs checkstyle on the java source
|
||||||
|
Validator("check_java_checkstyle", check_java_checkstyle, [],
|
||||||
|
os.path.join(OTTERTUNE_DIR, "script/formatting/config/google_checks.xml")),
|
||||||
|
|
||||||
|
# Checks that the python/java source files do not use illegal patterns
|
||||||
|
Validator('check_illegal_patterns', check_illegal_patterns, [], None),
|
||||||
|
|
||||||
|
# Checks that the python/java source files have headers
|
||||||
|
Validator('check_header', check_header, [], None)
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ==============================================
|
||||||
|
# MAIN FUNCTION
|
||||||
|
# ==============================================
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser(description="Validate OtterTune's source code")
|
||||||
|
parser.add_argument('paths', metavar='PATH', type=str, nargs='*',
|
||||||
|
help='Files or directories to (recursively) validate')
|
||||||
|
parser.add_argument('--staged-files', action='store_true',
|
||||||
|
help='Apply the selected action(s) to all staged files (git)')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
LOG.info('\nRunning source validators:\n%s\n',
|
||||||
|
'\n'.join(' ' + v.name for v in VALIDATORS))
|
||||||
|
for validator in VALIDATORS:
|
||||||
|
if not validate_validator(validator.modules, validator.config_path):
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
|
||||||
|
if args.staged_files:
|
||||||
|
targets = [os.path.abspath(os.path.join(OTTERTUNE_DIR, f))
|
||||||
|
for f in subprocess.check_output(["git", "diff", "--name-only", "HEAD",
|
||||||
|
"--cached", "--diff-filter=d"]).split()]
|
||||||
|
if not targets:
|
||||||
|
LOG.error("No staged files or not calling from a repository. Exiting...")
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
elif args.paths:
|
||||||
|
targets = args.paths
|
||||||
|
else:
|
||||||
|
targets = DEFAULT_DIRS
|
||||||
|
|
||||||
|
for target in targets:
|
||||||
|
target = os.path.abspath(target)
|
||||||
|
if os.path.isfile(target):
|
||||||
|
LOG.debug("Scanning file: %s\n", target)
|
||||||
|
status = validate_file(target)
|
||||||
|
elif os.path.isdir(target):
|
||||||
|
LOG.debug("Scanning directory: %s\n", target)
|
||||||
|
status = validate_dir(target)
|
||||||
|
else:
|
||||||
|
LOG.error("%s isn't a file or directory", target)
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
|
||||||
|
if not status:
|
||||||
|
LOG.info(SEPARATOR + '\n')
|
||||||
|
LOG.info("Validation NOT successful\n")
|
||||||
|
sys.exit(EXIT_FAILURE)
|
||||||
|
|
||||||
|
LOG.info("Validation successful\n")
|
||||||
|
sys.exit(EXIT_SUCCESS)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
#
|
||||||
|
# OtterTune - __init__.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
|
@ -0,0 +1,19 @@
|
||||||
|
#
|
||||||
|
# OtterTune - base.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Oct 25, 2017
|
||||||
|
|
||||||
|
@author: dva
|
||||||
|
'''
|
||||||
|
|
||||||
|
from abc import ABCMeta, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
|
class ModelBase(object, metaclass=ABCMeta):
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def _reset(self):
|
||||||
|
pass
|
||||||
|
|
@ -0,0 +1,793 @@
|
||||||
|
#
|
||||||
|
# OtterTune - cluster.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Jul 4, 2016
|
||||||
|
|
||||||
|
@author: dva
|
||||||
|
'''
|
||||||
|
from abc import ABCMeta, abstractproperty
|
||||||
|
from collections import OrderedDict
|
||||||
|
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import copy
|
||||||
|
import numpy as np
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
from scipy.spatial.distance import cdist
|
||||||
|
from sklearn.metrics import silhouette_score
|
||||||
|
from sklearn.cluster import KMeans as SklearnKMeans
|
||||||
|
from celery.utils.log import get_task_logger
|
||||||
|
|
||||||
|
from .base import ModelBase
|
||||||
|
|
||||||
|
# Log debug messages
|
||||||
|
LOGGER = get_task_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class KMeans(ModelBase):
|
||||||
|
"""
|
||||||
|
KMeans:
|
||||||
|
|
||||||
|
Fits an Sklearn KMeans model to X.
|
||||||
|
|
||||||
|
|
||||||
|
See also
|
||||||
|
--------
|
||||||
|
http://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
n_clusters_ : int
|
||||||
|
The number of clusters, K
|
||||||
|
|
||||||
|
cluster_inertia_ : float
|
||||||
|
Sum of squared distances of samples to their closest cluster center
|
||||||
|
|
||||||
|
cluster_labels_ : array, [n_clusters_]
|
||||||
|
Labels indicating the membership of each point
|
||||||
|
|
||||||
|
cluster_centers_ : array, [n_clusters, n_features]
|
||||||
|
Coordinates of cluster centers
|
||||||
|
|
||||||
|
sample_labels_ : array, [n_samples]
|
||||||
|
Labels for each of the samples in X
|
||||||
|
|
||||||
|
sample_distances_ : array, [n_samples]
|
||||||
|
The distance between each sample point and its cluster's center
|
||||||
|
|
||||||
|
|
||||||
|
Constants
|
||||||
|
---------
|
||||||
|
SAMPLE_CUTOFF_ : int
|
||||||
|
If n_samples > SAMPLE_CUTOFF_ then sample distances
|
||||||
|
are NOT recorded
|
||||||
|
"""
|
||||||
|
|
||||||
|
SAMPLE_CUTOFF_ = 1000
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.model_ = None
|
||||||
|
self.n_clusters_ = None
|
||||||
|
self.sample_labels_ = None
|
||||||
|
self.sample_distances_ = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cluster_inertia_(self):
|
||||||
|
# Sum of squared distances of samples to their closest cluster center
|
||||||
|
return None if self.model_ is None else \
|
||||||
|
self.model_.inertia_
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cluster_labels_(self):
|
||||||
|
# Cluster membership labels for each point
|
||||||
|
return None if self.model_ is None else \
|
||||||
|
copy.deepcopy(self.model_.labels_)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def cluster_centers_(self):
|
||||||
|
# Coordinates of the cluster centers
|
||||||
|
return None if self.model_ is None else \
|
||||||
|
copy.deepcopy(self.model_.cluster_centers_)
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
self.model_ = None
|
||||||
|
self.n_clusters_ = None
|
||||||
|
self.sample_labels_ = None
|
||||||
|
self.sample_distances_ = None
|
||||||
|
|
||||||
|
def fit(self, X, K, sample_labels=None, estimator_params=None):
|
||||||
|
"""Fits a Sklearn KMeans model to X.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
K : int
|
||||||
|
The number of clusters.
|
||||||
|
|
||||||
|
sample_labels : array-like, shape (n_samples), optional
|
||||||
|
Labels for each of the samples in X.
|
||||||
|
|
||||||
|
estimator_params : dict, optional
|
||||||
|
The parameters to pass to the KMeans estimators.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
# Note: previously set n_init=50
|
||||||
|
self.model_ = SklearnKMeans(K)
|
||||||
|
if estimator_params is not None:
|
||||||
|
assert isinstance(estimator_params, dict)
|
||||||
|
self.model_.set_params(**estimator_params)
|
||||||
|
|
||||||
|
# Compute Kmeans model
|
||||||
|
self.model_.fit(X)
|
||||||
|
if sample_labels is None:
|
||||||
|
sample_labels = ["sample_{}".format(i) for i in range(X.shape[0])]
|
||||||
|
assert len(sample_labels) == X.shape[0]
|
||||||
|
self.sample_labels_ = np.array(sample_labels)
|
||||||
|
self.n_clusters_ = K
|
||||||
|
|
||||||
|
# Record sample label/distance from its cluster center
|
||||||
|
self.sample_distances_ = OrderedDict()
|
||||||
|
for cluster_label in range(self.n_clusters_):
|
||||||
|
assert cluster_label not in self.sample_distances_
|
||||||
|
member_rows = X[self.cluster_labels_ == cluster_label, :]
|
||||||
|
member_labels = self.sample_labels_[self.cluster_labels_ == cluster_label]
|
||||||
|
centroid = np.expand_dims(self.cluster_centers_[cluster_label], axis=0)
|
||||||
|
|
||||||
|
# "All clusters must have at least 1 member!"
|
||||||
|
if member_rows.shape[0] == 0:
|
||||||
|
return None
|
||||||
|
|
||||||
|
# Calculate distance between each member row and the current cluster
|
||||||
|
dists = np.empty(member_rows.shape[0])
|
||||||
|
dist_labels = []
|
||||||
|
for j, (row, label) in enumerate(zip(member_rows, member_labels)):
|
||||||
|
dists[j] = cdist(np.expand_dims(row, axis=0), centroid, "euclidean").squeeze()
|
||||||
|
dist_labels.append(label)
|
||||||
|
|
||||||
|
# Sort the distances/labels in ascending order
|
||||||
|
sort_order = np.argsort(dists)
|
||||||
|
dists = dists[sort_order]
|
||||||
|
dist_labels = np.array(dist_labels)[sort_order]
|
||||||
|
self.sample_distances_[cluster_label] = {
|
||||||
|
"sample_labels": dist_labels,
|
||||||
|
"distances": dists,
|
||||||
|
}
|
||||||
|
return self
|
||||||
|
|
||||||
|
def get_closest_samples(self):
|
||||||
|
"""Returns a list of the labels of the samples that are located closest
|
||||||
|
to their cluster's center.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
----------
|
||||||
|
closest_samples : list
|
||||||
|
A list of the sample labels that are located the closest to
|
||||||
|
their cluster's center.
|
||||||
|
"""
|
||||||
|
if self.sample_distances_ is None:
|
||||||
|
raise Exception("No model has been fit yet!")
|
||||||
|
|
||||||
|
return [samples['sample_labels'][0] for samples in list(self.sample_distances_.values())]
|
||||||
|
|
||||||
|
def get_memberships(self):
|
||||||
|
'''
|
||||||
|
Return the memberships in each cluster
|
||||||
|
'''
|
||||||
|
memberships = OrderedDict()
|
||||||
|
for cluster_label, samples in list(self.sample_distances_.items()):
|
||||||
|
memberships[cluster_label] = OrderedDict(
|
||||||
|
[(l, d) for l, d in zip(samples["sample_labels"], samples["distances"])])
|
||||||
|
return json.dumps(memberships, indent=4)
|
||||||
|
|
||||||
|
|
||||||
|
class KMeansClusters(ModelBase):
|
||||||
|
|
||||||
|
"""
|
||||||
|
KMeansClusters:
|
||||||
|
|
||||||
|
Fits a KMeans model to X for clusters in the range [min_cluster_, max_cluster_].
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
min_cluster_ : int
|
||||||
|
The minimum cluster size to fit a KMeans model to
|
||||||
|
|
||||||
|
max_cluster_ : int
|
||||||
|
The maximum cluster size to fit a KMeans model to
|
||||||
|
|
||||||
|
cluster_map_ : dict
|
||||||
|
A dictionary mapping the cluster size (K) to the KMeans
|
||||||
|
model fitted to X with K clusters
|
||||||
|
|
||||||
|
sample_labels_ : array, [n_samples]
|
||||||
|
Labels for each of the samples in X
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.min_cluster_ = None
|
||||||
|
self.max_cluster_ = None
|
||||||
|
self.cluster_map_ = None
|
||||||
|
self.sample_labels_ = None
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
self.min_cluster_ = None
|
||||||
|
self.max_cluster_ = None
|
||||||
|
self.cluster_map_ = None
|
||||||
|
self.sample_labels_ = None
|
||||||
|
|
||||||
|
def fit(self, X, min_cluster, max_cluster, sample_labels=None, estimator_params=None):
|
||||||
|
"""Fits a KMeans model to X for each cluster in the range [min_cluster, max_cluster].
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
min_cluster : int
|
||||||
|
The minimum cluster size to fit a KMeans model to.
|
||||||
|
|
||||||
|
max_cluster : int
|
||||||
|
The maximum cluster size to fit a KMeans model to.
|
||||||
|
|
||||||
|
sample_labels : array-like, shape (n_samples), optional
|
||||||
|
Labels for each of the samples in X.
|
||||||
|
|
||||||
|
estimator_params : dict, optional
|
||||||
|
The parameters to pass to the KMeans estimators.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
self.min_cluster_ = min_cluster
|
||||||
|
self.max_cluster_ = max_cluster
|
||||||
|
self.cluster_map_ = {}
|
||||||
|
if sample_labels is None:
|
||||||
|
sample_labels = ["sample_{}".format(i) for i in range(X.shape[1])]
|
||||||
|
self.sample_labels_ = sample_labels
|
||||||
|
for K in range(self.min_cluster_, self.max_cluster_ + 1):
|
||||||
|
tmp = KMeans().fit(X, K, self.sample_labels_, estimator_params)
|
||||||
|
if tmp is None: # Set maximum cluster
|
||||||
|
assert K > min_cluster, "min_cluster is too large for the model"
|
||||||
|
self.max_cluster_ = K - 1
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
self.cluster_map_[K] = tmp
|
||||||
|
|
||||||
|
return self
|
||||||
|
|
||||||
|
def save(self, savedir):
|
||||||
|
"""Saves the KMeans model results
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
savedir : string
|
||||||
|
Path to the directory to save the results in.
|
||||||
|
"""
|
||||||
|
if self.cluster_map_ is None:
|
||||||
|
raise Exception("No models have been fitted yet!")
|
||||||
|
|
||||||
|
cluster_map = OrderedDict()
|
||||||
|
inertias = []
|
||||||
|
for K, model in sorted(self.cluster_map_.items()):
|
||||||
|
cluster_map[K] = {
|
||||||
|
"cluster_inertia": model.cluster_inertia_,
|
||||||
|
"cluster_labels": model.cluster_labels_,
|
||||||
|
"cluster_centers": model.cluster_centers_,
|
||||||
|
}
|
||||||
|
inertias.append(model.cluster_inertia_)
|
||||||
|
|
||||||
|
# Save sum of squares plot (elbow curve)
|
||||||
|
fig = plt.figure()
|
||||||
|
plt.plot(list(cluster_map.keys()), inertias, '--o')
|
||||||
|
plt.xlabel("Number of clusters (K)")
|
||||||
|
plt.ylabel("Within sum of squares W_k")
|
||||||
|
plt.title("Within Sum of Squares vs. Number of Clusters")
|
||||||
|
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||||
|
savepath = os.path.join(savedir, "kmeans_sum_of_squares.pdf")
|
||||||
|
plt.savefig(savepath, bbox_inches="tight")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
# save cluster memberships
|
||||||
|
for K in range(self.min_cluster_, self.max_cluster_ + 1):
|
||||||
|
savepath = os.path.join(savedir,
|
||||||
|
"memberships_{}-clusters.json".format(K))
|
||||||
|
members = self.cluster_map_[K].get_memberships()
|
||||||
|
with open(savepath, "w") as f:
|
||||||
|
f.write(members)
|
||||||
|
|
||||||
|
|
||||||
|
class KSelection(ModelBase, metaclass=ABCMeta):
|
||||||
|
"""KSelection:
|
||||||
|
|
||||||
|
Abstract class for techniques that approximate the optimal
|
||||||
|
number of clusters (K).
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
optimal_num_clusters_ : int
|
||||||
|
An estimation of the optimal number of clusters K for
|
||||||
|
a KMeans model fit to X
|
||||||
|
clusters_ : array, [n_clusters]
|
||||||
|
The sizes of the clusters
|
||||||
|
|
||||||
|
name_ : string
|
||||||
|
The name of this technique
|
||||||
|
"""
|
||||||
|
|
||||||
|
NAME_ = None
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.optimal_num_clusters_ = None
|
||||||
|
self.clusters_ = None
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
self.optimal_num_clusters_ = None
|
||||||
|
self.clusters_ = None
|
||||||
|
|
||||||
|
@abstractproperty
|
||||||
|
def name_(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def save(self, savedir):
|
||||||
|
"""Saves the estimation of the optimal # of clusters.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
savedir : string
|
||||||
|
Path to the directory to save the results in.
|
||||||
|
"""
|
||||||
|
if self.optimal_num_clusters_ is None:
|
||||||
|
raise Exception("Optimal number of clusters has not been computed!")
|
||||||
|
|
||||||
|
# Save the computed optimal number of clusters
|
||||||
|
savepath = os.path.join(savedir, self.name_ + "_optimal_num_clusters.txt")
|
||||||
|
with open(savepath, "w") as f:
|
||||||
|
f.write(str(self.optimal_num_clusters_))
|
||||||
|
|
||||||
|
|
||||||
|
class GapStatistic(KSelection):
|
||||||
|
"""GapStatistic:
|
||||||
|
|
||||||
|
Approximates the optimal number of clusters (K).
|
||||||
|
|
||||||
|
|
||||||
|
References
|
||||||
|
----------
|
||||||
|
https://web.stanford.edu/~hastie/Papers/gap.pdf
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
optimal_num_clusters_ : int
|
||||||
|
An estimation of the optimal number of clusters K for
|
||||||
|
a KMeans model fit to X
|
||||||
|
|
||||||
|
clusters_ : array, [n_clusters]
|
||||||
|
The sizes of the clusters
|
||||||
|
|
||||||
|
name_ : string
|
||||||
|
The name of this technique
|
||||||
|
|
||||||
|
log_wks_ : array, [n_clusters]
|
||||||
|
The within-dispersion measures of X (log)
|
||||||
|
|
||||||
|
log_wkbs_ : array, [n_clusters]
|
||||||
|
The within-dispersion measures of the generated
|
||||||
|
reference data sets
|
||||||
|
|
||||||
|
khats_ : array, [n_clusters]
|
||||||
|
The gap-statistic for each cluster
|
||||||
|
"""
|
||||||
|
|
||||||
|
NAME_ = "gap-statistic"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(GapStatistic, self).__init__()
|
||||||
|
self.log_wks_ = None
|
||||||
|
self.log_wkbs_ = None
|
||||||
|
self.khats_ = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name_(self):
|
||||||
|
return self.NAME_
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
super(GapStatistic, self)._reset()
|
||||||
|
self.log_wks_ = None
|
||||||
|
self.log_wkbs_ = None
|
||||||
|
self.khats_ = None
|
||||||
|
|
||||||
|
def fit(self, X, cluster_map, n_b=50):
|
||||||
|
"""Estimates the optimal number of clusters (K) for a
|
||||||
|
KMeans model trained on X.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
cluster_map_ : dict
|
||||||
|
A dictionary mapping each cluster size (K) to the KMeans
|
||||||
|
model fitted to X with K clusters
|
||||||
|
|
||||||
|
n_B : int
|
||||||
|
The number of reference data sets to generate
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
mins, maxs = GapStatistic.bounding_box(X)
|
||||||
|
n_clusters = len(cluster_map)
|
||||||
|
|
||||||
|
# Dispersion for real distribution
|
||||||
|
log_wks = np.zeros(n_clusters)
|
||||||
|
log_wkbs = np.zeros(n_clusters)
|
||||||
|
sk = np.zeros(n_clusters)
|
||||||
|
for indk, (K, model) in enumerate(sorted(cluster_map.items())):
|
||||||
|
|
||||||
|
# Computes Wk: the within-dispersion of each cluster size (k)
|
||||||
|
log_wks[indk] = np.log(model.cluster_inertia_ / (2.0 * K))
|
||||||
|
|
||||||
|
# Create B reference datasets
|
||||||
|
log_bwkbs = np.zeros(n_b)
|
||||||
|
for i in range(n_b):
|
||||||
|
Xb = np.empty_like(X)
|
||||||
|
for j in range(X.shape[1]):
|
||||||
|
Xb[:, j] = np.random.uniform(mins[j], maxs[j], size=X.shape[0])
|
||||||
|
Xb_model = KMeans().fit(Xb, K)
|
||||||
|
log_bwkbs[i] = np.log(Xb_model.cluster_inertia_ / (2.0 * K))
|
||||||
|
log_wkbs[indk] = sum(log_bwkbs) / n_b
|
||||||
|
sk[indk] = np.sqrt(sum((log_bwkbs - log_wkbs[indk]) ** 2) / n_b)
|
||||||
|
sk = sk * np.sqrt(1 + 1.0 / n_b)
|
||||||
|
|
||||||
|
khats = np.zeros(n_clusters)
|
||||||
|
gaps = log_wkbs - log_wks
|
||||||
|
gsks = gaps - sk
|
||||||
|
khats[1:] = gaps[0:-1] - gsks[1:]
|
||||||
|
self.clusters_ = np.array(sorted(cluster_map.keys()))
|
||||||
|
|
||||||
|
for i in range(1, n_clusters):
|
||||||
|
if gaps[i - 1] >= gsks[i]:
|
||||||
|
self.optimal_num_clusters_ = self.clusters_[i - 1]
|
||||||
|
break
|
||||||
|
|
||||||
|
if self.optimal_num_clusters_ is None:
|
||||||
|
LOGGER.info("GapStatistic NOT found the optimal k, \
|
||||||
|
use the last(maximum) k instead ")
|
||||||
|
self.optimal_num_clusters_ = self.clusters_[-1]
|
||||||
|
|
||||||
|
self.log_wks_ = log_wks
|
||||||
|
self.log_wkbs_ = log_wkbs
|
||||||
|
self.khats_ = khats
|
||||||
|
return self
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def bounding_box(X):
|
||||||
|
"""Computes the box that tightly bounds X
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The mins and maxs that make up the bounding box
|
||||||
|
"""
|
||||||
|
mins = np.min(X, axis=0)
|
||||||
|
maxs = np.max(X, axis=0)
|
||||||
|
return mins, maxs
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def Wk(X, mu, cluster_labels):
|
||||||
|
"""Computes the within-dispersion of each cluster size (k)
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
mu : array-like, shape (n_clusters, n_features)
|
||||||
|
Coordinates of cluster centers
|
||||||
|
|
||||||
|
cluster_labels: array-like, shape (n_samples)
|
||||||
|
Labels for each of the samples in X.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The within-dispersion of each cluster (K)
|
||||||
|
"""
|
||||||
|
K = len(mu)
|
||||||
|
return sum([np.linalg.norm(mu[i] - x) ** 2 / (2.0 * K)
|
||||||
|
for i in range(K)
|
||||||
|
for x in X[cluster_labels == i]])
|
||||||
|
|
||||||
|
def save(self, savedir):
|
||||||
|
"""Saves the estimation results of the optimal # of clusters.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
savedir : string
|
||||||
|
Path to the directory to save the results in.
|
||||||
|
"""
|
||||||
|
super(GapStatistic, self).save(savedir)
|
||||||
|
|
||||||
|
# Plot the calculated gap
|
||||||
|
gaps = self.log_wkbs_ - self.log_wks_
|
||||||
|
fig = plt.figure()
|
||||||
|
plt.plot(self.clusters_, gaps, '--o')
|
||||||
|
plt.title("Gap vs. Number of Clusters")
|
||||||
|
plt.xlabel("Number of clusters (K)")
|
||||||
|
plt.ylabel("gap_K")
|
||||||
|
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||||
|
plt.savefig(os.path.join(savedir, self.name_ + ".pdf"), bbox_inches="tight")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
# Plot the gap statistic
|
||||||
|
fig = plt.figure()
|
||||||
|
plt.bar(self.clusters_, self.khats_)
|
||||||
|
plt.title("Gap Statistic vs. Number of Clusters")
|
||||||
|
plt.xlabel("Number of clusters (K)")
|
||||||
|
plt.ylabel("gap(K)-(gap(K+1)-s(K+1))")
|
||||||
|
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||||
|
plt.savefig(os.path.join(savedir, self.name_ + "_final.pdf"),
|
||||||
|
bbox_inches="tight")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
|
class DetK(KSelection):
|
||||||
|
"""DetK:
|
||||||
|
|
||||||
|
Approximates the optimal number of clusters (K).
|
||||||
|
|
||||||
|
|
||||||
|
References
|
||||||
|
----------
|
||||||
|
https://www.ee.columbia.edu/~dpwe/papers/PhamDN05-kmeans.pdf
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
optimal_num_clusters_ : int
|
||||||
|
An estimation of the optimal number of clusters K for
|
||||||
|
KMeans models fit to X
|
||||||
|
|
||||||
|
clusters_ : array, [n_clusters]
|
||||||
|
The sizes of the clusters
|
||||||
|
|
||||||
|
name_ : string
|
||||||
|
The name of this technique
|
||||||
|
|
||||||
|
fs_ : array, [n_clusters]
|
||||||
|
The computed evaluation functions F(K) for each cluster size K
|
||||||
|
"""
|
||||||
|
|
||||||
|
NAME_ = "det-k"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(DetK, self).__init__()
|
||||||
|
self.fs_ = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name_(self):
|
||||||
|
return DetK.NAME_
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
super(DetK, self)._reset()
|
||||||
|
self.fs_ = None
|
||||||
|
|
||||||
|
def fit(self, X, cluster_map):
|
||||||
|
"""Estimates the optimal number of clusters (K) for a
|
||||||
|
KMeans model trained on X.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
cluster_map_ : dict
|
||||||
|
A dictionary mapping each cluster size (K) to the KMeans
|
||||||
|
model fitted to X with K clusters
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
n_clusters = len(cluster_map)
|
||||||
|
nd = X.shape[1]
|
||||||
|
fs = np.empty(n_clusters)
|
||||||
|
sks = np.empty(n_clusters)
|
||||||
|
alpha = {}
|
||||||
|
# K from 1 to maximum_cluster_
|
||||||
|
for i, (K, model) \
|
||||||
|
in enumerate(sorted(cluster_map.items())):
|
||||||
|
# Compute alpha(K, nd) (i.e. alpha[K])
|
||||||
|
if K == 2:
|
||||||
|
alpha[K] = 1 - 3.0 / (4 * nd)
|
||||||
|
elif K > 2:
|
||||||
|
alpha[K] = alpha[K - 1] + (1 - alpha[K - 1]) / 6.0
|
||||||
|
sks[i] = model.cluster_inertia_
|
||||||
|
|
||||||
|
if K == 1:
|
||||||
|
fs[i] = 1
|
||||||
|
elif sks[i - 1] == 0:
|
||||||
|
fs[i] = 1
|
||||||
|
else:
|
||||||
|
fs[i] = sks[i] / (alpha[K] * sks[i - 1])
|
||||||
|
self.clusters_ = np.array(sorted(cluster_map.keys()))
|
||||||
|
self.optimal_num_clusters_ = self.clusters_[np.argmin(fs)]
|
||||||
|
self.fs_ = fs
|
||||||
|
return self
|
||||||
|
|
||||||
|
def save(self, savedir):
|
||||||
|
"""Saves the estimation results of the optimal # of clusters.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
savedir : string
|
||||||
|
Path to the directory to save the results in.
|
||||||
|
"""
|
||||||
|
super(DetK, self).save(savedir)
|
||||||
|
|
||||||
|
# Plot the evaluation function
|
||||||
|
fig = plt.figure()
|
||||||
|
plt.plot(self.clusters_, self.fs_, '--o')
|
||||||
|
plt.xlabel("Number of clusters (K)")
|
||||||
|
plt.ylabel("Evaluation function (F_k)")
|
||||||
|
plt.title("Evaluation Function vs. Number of Clusters")
|
||||||
|
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||||
|
savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf")
|
||||||
|
plt.savefig(savepath, bbox_inches="tight")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
|
class Silhouette(KSelection):
|
||||||
|
"""Det:
|
||||||
|
|
||||||
|
Approximates the optimal number of clusters (K).
|
||||||
|
|
||||||
|
|
||||||
|
References
|
||||||
|
----------
|
||||||
|
http://scikit-learn.org/stable/modules/generated/sklearn.metrics.silhouette_score.html
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
optimal_num_clusters_ : int
|
||||||
|
An estimation of the optimal number of clusters K for
|
||||||
|
KMeans models fit to X
|
||||||
|
|
||||||
|
clusters_ : array, [n_clusters]
|
||||||
|
The sizes of the clusters
|
||||||
|
|
||||||
|
name_ : string
|
||||||
|
The name of this technique
|
||||||
|
|
||||||
|
Score_ : array, [n_clusters]
|
||||||
|
The mean Silhouette Coefficient for each cluster size K
|
||||||
|
"""
|
||||||
|
|
||||||
|
# short for Silhouette score
|
||||||
|
NAME_ = "s-score"
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
super(Silhouette, self).__init__()
|
||||||
|
self.scores_ = None
|
||||||
|
|
||||||
|
@property
|
||||||
|
def name_(self):
|
||||||
|
return Silhouette.NAME_
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
super(Silhouette, self)._reset()
|
||||||
|
self.scores_ = None
|
||||||
|
|
||||||
|
def fit(self, X, cluster_map):
|
||||||
|
"""Estimates the optimal number of clusters (K) for a
|
||||||
|
KMeans model trained on X.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
cluster_map_ : dict
|
||||||
|
A dictionary mapping each cluster size (K) to the KMeans
|
||||||
|
model fitted to X with K clusters
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
n_clusters = len(cluster_map)
|
||||||
|
# scores = np.empty(n_clusters)
|
||||||
|
scores = np.zeros(n_clusters)
|
||||||
|
for i, (K, model) \
|
||||||
|
in enumerate(sorted(cluster_map.items())):
|
||||||
|
if K <= 1: # K >= 2
|
||||||
|
continue
|
||||||
|
scores[i] = silhouette_score(X, model.cluster_labels_)
|
||||||
|
|
||||||
|
self.clusters_ = np.array(sorted(cluster_map.keys()))
|
||||||
|
self.optimal_num_clusters_ = self.clusters_[np.argmax(scores)]
|
||||||
|
self.scores_ = scores
|
||||||
|
return self
|
||||||
|
|
||||||
|
def save(self, savedir):
|
||||||
|
"""Saves the estimation results of the optimal # of clusters.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
savedir : string
|
||||||
|
Path to the directory to save the results in.
|
||||||
|
"""
|
||||||
|
super(Silhouette, self).save(savedir)
|
||||||
|
|
||||||
|
# Plot the evaluation function
|
||||||
|
fig = plt.figure()
|
||||||
|
plt.plot(self.clusters_, self.scores_, '--o')
|
||||||
|
plt.xlabel("Number of clusters (K)")
|
||||||
|
plt.ylabel("Silhouette scores")
|
||||||
|
plt.title("Silhouette Scores vs. Number of Clusters")
|
||||||
|
fig.canvas.set_window_title(os.path.basename(savedir))
|
||||||
|
savepath = os.path.join(savedir, self.name_ + "_eval_function.pdf")
|
||||||
|
plt.savefig(savepath, bbox_inches="tight")
|
||||||
|
plt.close()
|
||||||
|
|
||||||
|
|
||||||
|
def create_kselection_model(model_name):
|
||||||
|
"""Constructs the KSelection model object with the given name
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
model_name : string
|
||||||
|
Name of the KSelection model.
|
||||||
|
One of ['gap-statistic', 'det-k', 's-score']
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
The constructed model object
|
||||||
|
"""
|
||||||
|
kselection_map = {
|
||||||
|
DetK.NAME_: DetK,
|
||||||
|
GapStatistic.NAME_: GapStatistic,
|
||||||
|
Silhouette.NAME_: Silhouette
|
||||||
|
}
|
||||||
|
if model_name not in kselection_map:
|
||||||
|
raise Exception("KSelection model {} not supported!".format(model_name))
|
||||||
|
else:
|
||||||
|
return kselection_map[model_name]()
|
||||||
|
|
@ -0,0 +1,115 @@
|
||||||
|
#
|
||||||
|
# OtterTune - constraints.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Sep 8, 2016
|
||||||
|
|
||||||
|
@author: dvanaken
|
||||||
|
'''
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
|
||||||
|
class ParamConstraintHelper(object):
|
||||||
|
|
||||||
|
def __init__(self, scaler, encoder=None, binary_vars=None,
|
||||||
|
init_flip_prob=0.3, flip_prob_decay=0.5):
|
||||||
|
if 'inverse_transform' not in dir(scaler):
|
||||||
|
raise Exception("Scaler object must provide function inverse_transform(X)")
|
||||||
|
if 'transform' not in dir(scaler):
|
||||||
|
raise Exception("Scaler object must provide function transform(X)")
|
||||||
|
self.scaler_ = scaler
|
||||||
|
if encoder is not None and len(encoder.n_values) > 0:
|
||||||
|
self.is_dummy_encoded_ = True
|
||||||
|
self.encoder_ = encoder.encoder
|
||||||
|
else:
|
||||||
|
self.is_dummy_encoded_ = False
|
||||||
|
self.binary_vars_ = binary_vars
|
||||||
|
self.init_flip_prob_ = init_flip_prob
|
||||||
|
self.flip_prob_decay_ = flip_prob_decay
|
||||||
|
|
||||||
|
def apply_constraints(self, sample, scaled=True, rescale=True):
|
||||||
|
conv_sample = self._handle_scaling(sample, scaled)
|
||||||
|
|
||||||
|
if self.is_dummy_encoded_:
|
||||||
|
# apply categorical (ie enum var, >=3 values) constraints
|
||||||
|
n_values = self.encoder_.n_values_
|
||||||
|
cat_start_indices = self.encoder_.feature_indices_
|
||||||
|
for i, nvals in enumerate(n_values):
|
||||||
|
start_idx = cat_start_indices[i]
|
||||||
|
cvals = conv_sample[start_idx: start_idx + nvals]
|
||||||
|
cvals = np.array(np.arange(nvals) == np.argmax(cvals), dtype=float)
|
||||||
|
assert np.sum(cvals) == 1
|
||||||
|
conv_sample[start_idx: start_idx + nvals] = cvals
|
||||||
|
|
||||||
|
# apply binary (0-1) constraints
|
||||||
|
if self.binary_vars_ is not None:
|
||||||
|
for i in self.binary_vars_:
|
||||||
|
# round to closest
|
||||||
|
if conv_sample[i] >= 0.5:
|
||||||
|
conv_sample[i] = 1
|
||||||
|
else:
|
||||||
|
conv_sample[i] = 0
|
||||||
|
|
||||||
|
conv_sample = self._handle_rescaling(conv_sample, rescale)
|
||||||
|
return conv_sample
|
||||||
|
|
||||||
|
def _handle_scaling(self, sample, scaled):
|
||||||
|
if scaled:
|
||||||
|
if sample.ndim == 1:
|
||||||
|
sample = sample.reshape(1, -1)
|
||||||
|
sample = self.scaler_.inverse_transform(sample).ravel()
|
||||||
|
else:
|
||||||
|
sample = np.array(sample)
|
||||||
|
return sample
|
||||||
|
|
||||||
|
def _handle_rescaling(self, sample, rescale):
|
||||||
|
if rescale:
|
||||||
|
if sample.ndim == 1:
|
||||||
|
sample = sample.reshape(1, -1)
|
||||||
|
return self.scaler_.transform(sample).ravel()
|
||||||
|
return sample
|
||||||
|
|
||||||
|
def randomize_categorical_features(self, sample, scaled=True, rescale=True):
|
||||||
|
# If there are no categorical features, this function is a no-op.
|
||||||
|
if not self.is_dummy_encoded_:
|
||||||
|
return sample
|
||||||
|
n_values = self.encoder_.n_values_
|
||||||
|
cat_start_indices = self.encoder_.feature_indices_
|
||||||
|
n_cat_feats = len(n_values)
|
||||||
|
|
||||||
|
conv_sample = self._handle_scaling(sample, scaled)
|
||||||
|
flips = np.zeros((n_cat_feats,), dtype=bool)
|
||||||
|
|
||||||
|
# Always flip at least one categorical feature
|
||||||
|
flips[0] = True
|
||||||
|
|
||||||
|
# Flip the rest with decreasing probability
|
||||||
|
p = self.init_flip_prob_
|
||||||
|
for i in range(1, n_cat_feats):
|
||||||
|
if np.random.rand() <= p:
|
||||||
|
flips[i] = True
|
||||||
|
p *= self.flip_prob_decay_
|
||||||
|
|
||||||
|
flip_shuffle_indices = np.random.choice(np.arange(n_cat_feats),
|
||||||
|
n_cat_feats,
|
||||||
|
replace=False)
|
||||||
|
flips = flips[flip_shuffle_indices]
|
||||||
|
|
||||||
|
for i, nvals in enumerate(n_values):
|
||||||
|
if flips[i]:
|
||||||
|
start_idx = cat_start_indices[i]
|
||||||
|
current_val = conv_sample[start_idx: start_idx + nvals]
|
||||||
|
assert np.all(np.logical_or(current_val == 0, current_val == 1)), \
|
||||||
|
"categorical {0}: value not 0/1: {1}".format(i, current_val)
|
||||||
|
choices = np.arange(nvals)[current_val != 1]
|
||||||
|
assert choices.size == nvals - 1
|
||||||
|
r = np.zeros(nvals)
|
||||||
|
r[np.random.choice(choices)] = 1
|
||||||
|
assert np.sum(r) == 1
|
||||||
|
conv_sample[start_idx: start_idx + nvals] = r
|
||||||
|
|
||||||
|
conv_sample = self._handle_rescaling(conv_sample, rescale)
|
||||||
|
return conv_sample
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
#
|
||||||
|
# OtterTune - factor_analysis.py
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017-18, Carnegie Mellon University Database Group
|
||||||
|
#
|
||||||
|
'''
|
||||||
|
Created on Jul 4, 2016
|
||||||
|
|
||||||
|
@author: dvanaken
|
||||||
|
'''
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
from sklearn.decomposition import FactorAnalysis as SklearnFactorAnalysis
|
||||||
|
|
||||||
|
from .base import ModelBase
|
||||||
|
|
||||||
|
|
||||||
|
class FactorAnalysis(ModelBase):
|
||||||
|
"""FactorAnalysis (FA):
|
||||||
|
|
||||||
|
Fits an Sklearn FactorAnalysis model to X.
|
||||||
|
|
||||||
|
|
||||||
|
See also
|
||||||
|
--------
|
||||||
|
http://scikit-learn.org/stable/modules/generated/sklearn.decomposition.FactorAnalysis.html
|
||||||
|
|
||||||
|
|
||||||
|
Attributes
|
||||||
|
----------
|
||||||
|
model_ : sklearn.decomposition.FactorAnalysis
|
||||||
|
The fitted FA model
|
||||||
|
|
||||||
|
components_ : array, [n_components, n_features]
|
||||||
|
Components (i.e., factors) with maximum variance
|
||||||
|
|
||||||
|
feature_labels_ : array, [n_features]
|
||||||
|
|
||||||
|
total_variance_ : float
|
||||||
|
The total amount of variance explained by the components
|
||||||
|
|
||||||
|
pvars_ : array, [n_components]
|
||||||
|
The percentage of the variance explained by each component
|
||||||
|
|
||||||
|
pvars_noise_ : array, [n_components]
|
||||||
|
The percentage of the variance explained by each component also
|
||||||
|
accounting for noise
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.model_ = None
|
||||||
|
self.components_ = None
|
||||||
|
self.feature_labels_ = None
|
||||||
|
self.total_variance_ = None
|
||||||
|
self.pvars_ = None
|
||||||
|
self.pvars_noise_ = None
|
||||||
|
|
||||||
|
def _reset(self):
|
||||||
|
"""Resets all attributes (erases the model)"""
|
||||||
|
self.model_ = None
|
||||||
|
self.components_ = None
|
||||||
|
self.feature_labels_ = None
|
||||||
|
self.total_variance_ = None
|
||||||
|
self.pvars_ = None
|
||||||
|
self.pvars_noise_ = None
|
||||||
|
|
||||||
|
def fit(self, X, feature_labels=None, n_components=None, estimator_params=None):
|
||||||
|
"""Fits an Sklearn FA model to X.
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
X : array-like, shape (n_samples, n_features)
|
||||||
|
Training data.
|
||||||
|
|
||||||
|
feature_labels : array-like, shape (n_features), optional
|
||||||
|
Labels for each of the features in X.
|
||||||
|
|
||||||
|
estimator_params : dict, optional
|
||||||
|
The parameters to pass to Sklearn's FA estimators.
|
||||||
|
|
||||||
|
|
||||||
|
Returns
|
||||||
|
-------
|
||||||
|
self
|
||||||
|
"""
|
||||||
|
self._reset()
|
||||||
|
if feature_labels is None:
|
||||||
|
feature_labels = ["feature_{}".format(i) for i in range(X.shape[1])]
|
||||||
|
self.feature_labels_ = feature_labels
|
||||||
|
if n_components is not None:
|
||||||
|
model = SklearnFactorAnalysis(n_components=n_components)
|
||||||
|
else:
|
||||||
|
model = SklearnFactorAnalysis()
|
||||||
|
self.model_ = model
|
||||||
|
if estimator_params is not None:
|
||||||
|
# Update Sklearn estimator params
|
||||||
|
assert isinstance(estimator_params, dict)
|
||||||
|
self.model_.set_params(**estimator_params)
|
||||||
|
self.model_.fit(X)
|
||||||
|
|
||||||
|
# Remove zero-valued components (n_components x n_features)
|
||||||
|
components_mask = np.sum(self.model_.components_ != 0.0, axis=1) > 0.0
|
||||||
|
self.components_ = self.model_.components_[components_mask]
|
||||||
|
|
||||||
|
# Compute the % variance explained (with/without noise)
|
||||||
|
c2 = np.sum(self.components_ ** 2, axis=1)
|
||||||
|
self.total_variance_ = np.sum(c2)
|
||||||
|
self.pvars_ = 100 * c2 / self.total_variance_
|
||||||
|
self.pvars_noise_ = 100 * c2 / (self.total_variance_ +
|
||||||
|
np.sum(self.model_.noise_variance_))
|
||||||
|
return self
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue