add data types, logs, sequence mining to notecard.
This commit is contained in:
matt
commit
38c49616a2
Binary file not shown.
|
|
@ -0,0 +1,432 @@
|
||||||
|
\documentclass[sigconf,authorversion,nonacm]{acmart}
|
||||||
|
|
||||||
|
\usepackage{bbold}
|
||||||
|
\usepackage{geometry}
|
||||||
|
\geometry{margin=0.3in}
|
||||||
|
|
||||||
|
\nonstopmode
|
||||||
|
|
||||||
|
\begin{document}
|
||||||
|
\title{CSCI577 Final}
|
||||||
|
|
||||||
|
\maketitle
|
||||||
|
|
||||||
|
\section{Data Types}
|
||||||
|
\begin{table*}
|
||||||
|
\caption{data types}
|
||||||
|
\begin{tabular}{lll}
|
||||||
|
\toprule
|
||||||
|
Variable type & Description & Examples \\
|
||||||
|
\midrule
|
||||||
|
Categorical & & \\
|
||||||
|
Nominal (unordered) & Gives only qualitative information & Names, occupations, nationalities, sex, religion\\
|
||||||
|
Ordinal (ordered) & Ranking or order is important & Social status, economic class \\
|
||||||
|
Numeric & & \\
|
||||||
|
Interval & Distance between values has meaning (discrete or continuous) & Year, temperature \\
|
||||||
|
Ratio & Ratio of two values has meaning & Wealth, age, prices, wages \\
|
||||||
|
\bottomrule
|
||||||
|
\end{tabular}
|
||||||
|
\end{table*}
|
||||||
|
|
||||||
|
\section{Math}
|
||||||
|
|
||||||
|
\begin{displaymath}
|
||||||
|
log_{b}a = \frac{log_{x}a}{log_{x}b}
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
\section{Classification}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Constructing a method of classifying new instances using information in a training set
|
||||||
|
|
||||||
|
\begin{itemize}
|
||||||
|
\item Naive Bayes (Conditional Probabilities)
|
||||||
|
\item Decision Trees
|
||||||
|
\item Logistic Regression
|
||||||
|
\item Neural Networks
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
\subsection{Naïve Bayes}
|
||||||
|
Bayes Theorem:
|
||||||
|
\begin{displaymath}
|
||||||
|
P(A|B) = \frac{P(A,B)}{P(B)} = \frac{P(B|A) \times P(A}{P(B)}
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
\paragraph{Calculation}
|
||||||
|
Probablity of a class given attributes is a product of probability of that class overall, with the sum product of each individual attribute given the class:
|
||||||
|
\begin{displaymath}
|
||||||
|
P(c_{i} | v) = P(c_{i}) \prod_{j=1}^{n} P(a_{j} = v_{j} | \text{class} = c_{i})
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
|
||||||
|
\subsection{TDIDT}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Top-Down Induction of Decision Trees
|
||||||
|
|
||||||
|
\paragraph{Algorithm}
|
||||||
|
Until no more splitting is possible:
|
||||||
|
\begin{itemize}
|
||||||
|
\item IF all the instances in the training set belong to the same class THEN return the value of the class
|
||||||
|
\item ELSE
|
||||||
|
\begin{enumerate}
|
||||||
|
\item (a) Select an attribute A to split on
|
||||||
|
\item (b) Sort the instances in the training set into subsets, one for each value of attribute A
|
||||||
|
\item (c) Return a tree with one branch for each non-empty subset
|
||||||
|
\begin{itemize}
|
||||||
|
\item Each branch having a descendant subtree or a class value produced by applying the algorithm recursively
|
||||||
|
\end{itemize}
|
||||||
|
\end{enumerate}
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
|
||||||
|
\subsection{Adequacy}
|
||||||
|
\paragraph{Definition}
|
||||||
|
No two instances with the same values of all the attributes may belong to different classes.
|
||||||
|
Naive bayes can still be used when this doesn't obtain, as it will still be able to obtain the probabilities of each class.
|
||||||
|
kNN can still be used, as long as then multiple datapoints in the same location in euclidean space would still function as expected.
|
||||||
|
|
||||||
|
\subsection{Overfitting}
|
||||||
|
Understand the concept of overfitting and be able to tell how you would know that a classification system overfit
|
||||||
|
\paragraph{Definition}
|
||||||
|
If classifier generates a decision tree (or other mechanism) too well adapted to the training set
|
||||||
|
Performs well on training set, not well on other data.
|
||||||
|
Some overfitting inevitable.
|
||||||
|
|
||||||
|
Remedy:
|
||||||
|
\begin{itemize}
|
||||||
|
\item Adjust a decision tree while it is being generated: Pre-pruning
|
||||||
|
\item Modify the tree after creation: Post-pruning
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
\subsubsection{Clashes}
|
||||||
|
Two (or more) instances of a training set have identical attribute values, but different classification.
|
||||||
|
Especially a problem for TDIDT's 'Adequacy condition'.
|
||||||
|
|
||||||
|
\paragraph{Stems from}
|
||||||
|
\begin{itemize}
|
||||||
|
\item Classification incorrectly recorded
|
||||||
|
\item Recorded attributes insufficient - Would need more attributes, normally impossible
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
\paragraph{Solutions}
|
||||||
|
\begin{itemize}
|
||||||
|
\item Discard the branch to the clashing node from the node above
|
||||||
|
\item Of clashing instances, assume the majority label
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
\subsubsection{Prepruning}
|
||||||
|
When pre-pruning, may reduce accuracy on training set but may be better on test set (and subsequent) data than unpruned classifier.
|
||||||
|
\begin{enumerate}
|
||||||
|
\item test whether a termination condition applies.
|
||||||
|
\begin{itemize}
|
||||||
|
\item If so, current subset is treated as a 'clash set'
|
||||||
|
\item Resolve by 'delete branch,' 'majority voting,' etc.
|
||||||
|
\end{itemize}
|
||||||
|
\item two methods methods
|
||||||
|
\begin{itemize}
|
||||||
|
\item Size Cuttoff – prune of subset has fewer than X instances
|
||||||
|
\item Maximum depth: prune if length of branch exceed Y
|
||||||
|
\end{itemize}
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
\subsubsection{PostPruning}
|
||||||
|
\begin{enumerate}
|
||||||
|
\item look for a non-leaf nodes that have descendants of length 1.
|
||||||
|
\item In this tree, only node G and D are candidates for pruning (consolidation).
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
|
||||||
|
\subsection{Discretizing}
|
||||||
|
|
||||||
|
\subsubsection{Equal Width Intervals}
|
||||||
|
\subsubsection{Pseudo Attributes}
|
||||||
|
\subsubsection{Processing Sorted Instance Table}
|
||||||
|
\subsubsection{ChiMerge}
|
||||||
|
|
||||||
|
\paragraph{Rationalization}
|
||||||
|
Initially, each distinct value of a numerical attribute $A$ is considered to be one interval.
|
||||||
|
$\chi^{2}$ tests are performed for every pair of adjacent intervals.
|
||||||
|
Adjacent intervals with the least $\chi^{2}$ values are merged together, because $\chi^{2}$ low values for a pair indicates similar class distributions.
|
||||||
|
This merging process proceeds recursively until a predefined stopping criterion is met.
|
||||||
|
For two adjacent intervals, if $\chi^{2}$ test concludes that the class is independent intervals should be merged.
|
||||||
|
If $\chi^{2}$ test concludes that they are not independent, i.e. the difference in relative class frequency is statistically significant, the two intervals should remain separate.
|
||||||
|
|
||||||
|
\paragraph{Calculation}
|
||||||
|
To calculate expected value for any combination of row and class:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Take the product of the corresponding row sum and column sum
|
||||||
|
\item Divided by the grand total of the observed values for the two rows.
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
Then:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Using observed and expected values, calculate, for each of the cells:
|
||||||
|
\begin{math}
|
||||||
|
\frac{(O - E)^{2}}{E}
|
||||||
|
\end{math}
|
||||||
|
\item Sum each cell's $\chi^{2}$
|
||||||
|
\end{enumerate}
|
||||||
|
When exceeds $\chi^{2}$ threshold, hypothesis is rejected.
|
||||||
|
Small value for supports hypothesis.
|
||||||
|
Important adjustment, when $E < 0.5$ replace it with $0.5$.
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Select the smallest value
|
||||||
|
\item Compare it to the threshold
|
||||||
|
\item If it falls below the threshold, merge it with the row immediately below it
|
||||||
|
\item recalculate $\chi^{2}$, Only need to do this for rows adjacent to the recently merged one.
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
Large numbers of intervals does little to solve the problem of discretization.
|
||||||
|
Just one interval cannot contribute to a decision making process.
|
||||||
|
Modify significance level hypothesis of independence must pass, triggering interval merge.
|
||||||
|
Set a minimum and a maximum number of intervals
|
||||||
|
|
||||||
|
\subsection{Entropy}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Entropy is the measure of the presence of there being more than one possible classification.
|
||||||
|
Used for splitting attributes in decision trees
|
||||||
|
Entropy minimizes the complexity, number of branches, in the decision tree.
|
||||||
|
No guarantee that using entropy will always lead to a small decision Tree
|
||||||
|
Used for feature reduction: Calculate the value of information gain for each attribute in the original dataset. Discard all attributes that do not meet a specified criterion.
|
||||||
|
Pass the revised dataset to the preferred classification algorithm
|
||||||
|
|
||||||
|
Entropy has bias towards selecting attributes with a large number of values
|
||||||
|
\paragraph{Calculation}
|
||||||
|
To decide if you split on an attribute:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item find the entropy of the data in each of the branches after the split
|
||||||
|
\item then take the average of those and use it to find information gain.
|
||||||
|
\item The attribute split with the highest information gain (lowest entropy) is selected.
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
\begin{itemize}
|
||||||
|
\item Entropy is always positive or zero
|
||||||
|
\item Entropy is zero when $p_{i} = 1$, aka when all instances have the same class
|
||||||
|
\item Entropy is at its max value for the \# of classes when all classes are evenly distributed
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
If there are classes, we can denote the proportion of instances with classification $i$ by $p_{i}$ for $i = 1 to K$.
|
||||||
|
$p_{i} = \frac{\text{instances of class} i}{\text{total number of instances}}$
|
||||||
|
|
||||||
|
\begin{displaymath}
|
||||||
|
\text{Entropy} = E = -\sum_{i=1}^{K} p_{i} log_{2} p_{i}
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
where $K =$ non-empty classes and $p_{i} = \frac{\lvert i \rvert}{N}$, instances in class $i$ over total number of instances $N$.
|
||||||
|
|
||||||
|
|
||||||
|
\subsection{GINI}
|
||||||
|
|
||||||
|
\paragraph{Calculation}
|
||||||
|
\begin{enumerate}
|
||||||
|
\item For each non-empty column, form the sum of the squares of the values in the body of the table and divide by the column sum.
|
||||||
|
\item Add the values obtained for all the columns and divide by N (the number of instances).
|
||||||
|
\item Subtract the total from 1.
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
\subsection{Information Gain}
|
||||||
|
\paragraph{Definition}
|
||||||
|
The difference between the entropy before and after splitting on a given attribute in a decision tree.
|
||||||
|
Maximizing information gain is the same as minimizing $E_{new}$.
|
||||||
|
\paragraph{Calculation}
|
||||||
|
|
||||||
|
\begin{displaymath}
|
||||||
|
\text{Information Gain} = E_{\text{start}} - E_{\text{new}}
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
Starting node:
|
||||||
|
|
||||||
|
\begin{eqnarray}
|
||||||
|
E_{\text{start}} = -\frac{4}{24}log_{2}\frac{4}{24} \\ \nonumber
|
||||||
|
-\frac{5}{24}log_{2}\frac{5}{24} \\ \nonumber
|
||||||
|
-\frac{15}{24}log_{2}\frac{15}{24}
|
||||||
|
\end{eqnarray}
|
||||||
|
|
||||||
|
After spliting on attribute:
|
||||||
|
|
||||||
|
\begin{eqnarray}
|
||||||
|
E_{\text{new}} = \frac{8}{24}E_{1}\\ \nonumber
|
||||||
|
+ \frac{8}{24}E_{2}\\ \nonumber
|
||||||
|
+ \frac{8}{24}E_{3}\\ \nonumber
|
||||||
|
\end{eqnarray}
|
||||||
|
|
||||||
|
|
||||||
|
\paragraph{Uses}
|
||||||
|
|
||||||
|
|
||||||
|
\section{Clustering}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Grouping data into seperate groups.
|
||||||
|
Use distance metric between two datapoints.
|
||||||
|
Groups should be distinct from another and composed of items similar to one another, and different from items in other groups.
|
||||||
|
|
||||||
|
\subsection{Nearest Neighbors}
|
||||||
|
Mainly used when all attribute values are continuous
|
||||||
|
|
||||||
|
General strategy:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Find the $k$ training instances that are closest to the unseen instance.
|
||||||
|
\item Take the most commonly occurring classification for these instances.
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
\begin{itemize}
|
||||||
|
\item KMeans
|
||||||
|
\item DBSCAN
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
|
||||||
|
\section{Sequence Mining}
|
||||||
|
\textbf{TODO}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Finding meaningful, recurring sequences of events
|
||||||
|
|
||||||
|
\begin{itemize}
|
||||||
|
\item A sequence is an ordered list of elements (transactions):
|
||||||
|
\begin{displaymath}
|
||||||
|
s = <e_1 e_2 e_3>
|
||||||
|
\end{displaymath}
|
||||||
|
\item Each element contains a collection of events (items):
|
||||||
|
\begin{displaymath}
|
||||||
|
e_i = {i_1 i_2 i_3 \cdots i_k}
|
||||||
|
\end{displaymath}
|
||||||
|
\item Each element is attributed to a specific time or location.
|
||||||
|
\item Length of a sequence, $\lvert s \rvert$, is given by the number of elements of the sequence.
|
||||||
|
\item A k-sequence is a sequence that contains k events (items)
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
\paragraph{Contains}
|
||||||
|
A sequence \begin{math} <a_1 a_2 \cdots a_n> \end{math}
|
||||||
|
is contained in another sequence \begin{math}<b_1 b_2 \cdots b_m>; (m \geq n)\end{math}
|
||||||
|
if there exist integers \begin{math} i_1 < i_2 < \cdots < i_n \end{math}
|
||||||
|
such that \begin{math} a_1 \subseteq b_{i1} a_2 \subseteq b_{i2} \cdots a_n \subseteq b_{in} \end{math}.
|
||||||
|
|
||||||
|
\paragraph{Support}
|
||||||
|
The support of a subsequence w is defined as the fraction of data sequences that contain w.
|
||||||
|
A sequential pattern is a frequent subsequence (i.e., a subsequence where $\text{support} \geq \text{minsup}$)
|
||||||
|
|
||||||
|
\subsection{Generalized Sequential Pattern}
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Make the first pass over the sequence database D to yield all the 1-element frequent sequences
|
||||||
|
\item Repeat until no new frequent sequences are found:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Candidate Generation: Merge pairs of frequent subsequences found in the $(k-1)$'th pass to generate candidate sequences that contain $k$ items
|
||||||
|
\item Initial Pruning: Prune if it is not the case that all of the $k-1$ subsequences of a $k$ sequence are frequent
|
||||||
|
\item Support Counting: Make a new pass over the sequence database $D$ to find the support for these candidate sequences
|
||||||
|
\item Candidate Elimination: Eliminate candidate k-sequences whose actual support is less than minsup
|
||||||
|
\end{enumerate}
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
\subsection{Counting Methods}
|
||||||
|
\begin{itemize}
|
||||||
|
\item COBJ: One occurrence per object
|
||||||
|
\item CWIN: One occurrence per sliding window
|
||||||
|
\item CMINWIN: Number of minimal windows of occurrence
|
||||||
|
\item CDIST 0: Distinct occurrences with possibility of event-timestamp overlap
|
||||||
|
\item CDIST: Distinct occurrences with no event- timestamp overlap allowed
|
||||||
|
\end{itemize}
|
||||||
|
|
||||||
|
|
||||||
|
\section{Association Rule Analysis}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Given a collection of collections (database of transactions of food items), find items with high co-occurance.
|
||||||
|
|
||||||
|
Let $m$ be the number possible items that can be bought.
|
||||||
|
Let $I$ denote the set of all possible items.
|
||||||
|
Possible itemsets: $s^{\lvert I \rvert}$
|
||||||
|
An itemset $S$ matches a transaction $T$ (itself an itemset) if $S \subset T$.
|
||||||
|
|
||||||
|
\subsection{Support}
|
||||||
|
|
||||||
|
\paragraph{Definition}
|
||||||
|
$support(S)$: proportion of itemsets matched by $S$.
|
||||||
|
Proportion of transactions that contain all the items in $S$.
|
||||||
|
Frequency with which the items in S occur together in the database.
|
||||||
|
|
||||||
|
\paragraph{Calculation}
|
||||||
|
\begin{displaymath}
|
||||||
|
\text{support}(S) = \frac{count(S)}{n}
|
||||||
|
\end{displaymath}
|
||||||
|
where n is the number of transactions in the database.
|
||||||
|
|
||||||
|
|
||||||
|
\subsubsection{Uses}
|
||||||
|
|
||||||
|
\subsection{Confidence}
|
||||||
|
|
||||||
|
\paragraph{Calculation}
|
||||||
|
Confidence of a rule can be calculated either by
|
||||||
|
\begin{displaymath}
|
||||||
|
Confidence(L \rightarrow R) = \frac{count(L \cup R)}{count(L)}
|
||||||
|
\end{displaymath}
|
||||||
|
or
|
||||||
|
\begin{displaymath}
|
||||||
|
Confidence(L \rightarrow R) = \frac{support(L \cup R)}{support(L)}
|
||||||
|
\end{displaymath}
|
||||||
|
Reject rules where
|
||||||
|
\begin{displaymath}
|
||||||
|
support < minsup \approx 0.01 = 1\%
|
||||||
|
\end{displaymath}
|
||||||
|
Also called a frequent|large|supported itemset.
|
||||||
|
|
||||||
|
Reject rules where
|
||||||
|
\begin{displaymath}
|
||||||
|
confidence < minconf \approx 0.8 = 80\%
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
\paragraph{Uses}
|
||||||
|
|
||||||
|
\subsection{Lift}
|
||||||
|
\paragraph{Definition}
|
||||||
|
Lift measures how many more times the items in and occur together than would be expected if they were statistically independent.
|
||||||
|
Although lift is a useful measure of interestingness it is not always the best one to use.
|
||||||
|
In some cases a rule with higher support and lower lift can be more interesting than one with lower support and higher lift because it applies to more cases.
|
||||||
|
\paragraph{Calculation}
|
||||||
|
|
||||||
|
\begin{eqnarray}
|
||||||
|
\text{Lift}(L \rightarrow R)
|
||||||
|
= \frac{\text{count}(L \cup R)}{\text{count}(L) \times \text{support}(R)} \nonumber
|
||||||
|
\\
|
||||||
|
= \frac{\text{support}(L \cup R)}{\text{support}(L) \times \text{support}(R)} \nonumber
|
||||||
|
\\
|
||||||
|
= \frac{\text{confidence}(L \rightarrow R)}{\text{support}(R)} \nonumber
|
||||||
|
\\
|
||||||
|
= \frac{N \times \text{confidence}(L \rightarrow R)}{\text{count}(R)} \nonumber
|
||||||
|
\\
|
||||||
|
= \frac{N \times \text{confidence}(R \rightarrow L)}{\text{count}(R)} \nonumber
|
||||||
|
\\
|
||||||
|
= \text{Lift}(R \rightarrow L)
|
||||||
|
\end{eqnarray}
|
||||||
|
|
||||||
|
\subsection{Leverage}
|
||||||
|
\paragraph{Calculation}
|
||||||
|
\begin{displaymath}
|
||||||
|
\text{leverage}(L \rightarrow R) = \text{support}(L \cup R) - \text{support}(L) \times \text{support}(R)
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
\subsection{Frequent Itemsets}
|
||||||
|
\begin{enumerate}
|
||||||
|
\item Find itemsets of size $k$ made from 2 supported itemsets of size $k-1$
|
||||||
|
\item For each new itemset:
|
||||||
|
\begin{enumerate}
|
||||||
|
\item check if every sub-itemset in it also exists in the supported itemsets of size $k - 1$.
|
||||||
|
\item If not every sub-itemset does, then prune it
|
||||||
|
\end{enumerate}
|
||||||
|
\item Now with the final candidates, determine if they have mininum support
|
||||||
|
\item To determine association rules, find which itemsets have at least minimum confidence
|
||||||
|
\end{enumerate}
|
||||||
|
|
||||||
|
|
||||||
|
\subsection{Rules Possible}
|
||||||
|
|
||||||
|
\begin{displaymath}
|
||||||
|
\sideset{_k}{_i}C
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
or
|
||||||
|
|
||||||
|
\begin{displaymath}
|
||||||
|
2^{k} - 2
|
||||||
|
\end{displaymath}
|
||||||
|
|
||||||
|
|
||||||
|
\end{document}
|
||||||
|
\endinput
|
||||||
Loading…
Reference in New Issue