% \documentclass[11pt]{article}
% \usepackage{amsmath, amssymb, booktabs, geometry, xcolor, graphicx}
% \geometry{margin=1in}
% \usepackage{array}
% \usepackage{titlesec}
% \usepackage{caption}
% \captionsetup[table]{skip=6pt}
% \usepackage{hyperref}
%
% \title{\textbf{Normalization in Machine Learning: When to Use What}}
% \author{}
% \date{}
%
% \begin{document}
% \maketitle
%
% \section*{1. Overview}
% Normalization (or standardization/scaling) ensures features are comparable in scale and distribution.
% It directly affects gradient-based optimization, distance-based metrics, and regularization.
% Different techniques suit different models and data distributions.
%
% \begin{center}
% \includegraphics[width=0.8\textwidth]{normalization_diagram.png}
% \end{center}
%
% \textit{(Conceptual illustration: raw features on different scales $\rightarrow$ normalized comparable scales.)}
%
% \newpage
% \section*{2. Common Normalizations}
%
% \begin{table}[h!]
% \centering
% \small
% \renewcommand{\arraystretch}{1.3}
% \begin{tabular}{p{3.2cm} p{3cm} p{3.3cm} p{5cm}}
% \toprule
% \textbf{Name} & \textbf{Formula} & \textbf{Typical Use} & \textbf{Notes} \\
% \midrule
% \textbf{Zero-mean unit-variance (Z-score)} & $x' = \frac{x - \mu}{\sigma}$ & Linear models, SVM, PCA, neural nets & Works well when data $\sim$ Gaussian; sensitive to outliers. \\
% \textbf{Min-max scaling} & $x' = \frac{x - x_{\min}}{x_{\max}-x_{\min}}$ & Distance-based models, bounded features & Preserves shape but can distort with outliers. \\
% \textbf{Mean normalization} & $x' = \frac{x - \mu}{x_{\max}-x_{\min}}$ & Similar to above & Centers around 0; keeps shape. \\
% \textbf{MaxAbs scaling} & $x' = \frac{x}{|x|_{\max}}$ & Sparse data (TF-IDF, embeddings) & Keeps sparsity pattern intact. \\
% \textbf{Robust scaling} & $x' = \frac{x - \text{median}(x)}{\text{IQR}(x)}$ & Heavy-tailed or skewed data & Uses median and interquartile range. \\
% \textbf{L2 normalization} & $\mathbf{x}' = \frac{\mathbf{x}}{\|\mathbf{x}\|_2}$ & Text or embedding vectors & Ensures unit-length vectors for cosine similarity. \\
% \textbf{Log / Power transform} & $x' = \log(x + c)$ & Exponential or skewed features & Reduces skewness; often combined with standardization. \\
% \textbf{Quantile / Rank transform} & $\text{Quantile} \mapsto \mathcal{N}(0,1)$ & Tree-based, ensemble, or robust pipelines & Equalizes marginal distributions. \\
% \bottomrule
% \end{tabular}
% \caption{Common normalization and transformation methods.}
% \end{table}
%
% \newpage
% \section*{3. Model-Specific Guidelines}
%
% \begin{table}[h!]
% \centering
% \small
% \renewcommand{\arraystretch}{1.3}
% \begin{tabular}{p{4cm} p{4cm} p{6cm}}
% \toprule
% \textbf{Model Type} & \textbf{Recommended Normalization} & \textbf{Reason} \\
% \midrule
% Linear regression, logistic regression, SVM & Z-score standardization & Ensures balanced gradient steps and regularization strength. \\
% Neural networks & Z-score or min-max scaling & Keeps activations within non-saturated ranges; improves stability. \\
% Tree-based models (RF, XGBoost, LightGBM) & None usually needed & Tree splits are order-based and scale-invariant. \\
% k-NN, k-means, PCA, distance-based models & Z-score or min-max & Distance metrics depend on feature magnitudes. \\
% Text / sparse vectors & MaxAbsScaler or L2 normalization & Preserves sparsity and vector norms. \\
% Probabilistic models (Naive Bayes, count models) & None or log-transform & Assumes counts or frequencies directly. \\
% \bottomrule
% \end{tabular}
% \caption{Normalization preferences by model family.}
% \end{table}
%
% \section*{4. Practical Heuristics}
%
% \begin{itemize}
% \item Different feature units (e.g., age, income): \textbf{StandardScaler}
% \item Extreme outliers: \textbf{RobustScaler} or log-transform + StandardScaler
% \item Features bounded in $[0,1]$: \textbf{MinMaxScaler}
% \item Sparse text data: \textbf{MaxAbsScaler} or \textbf{L2 normalization}
% \item Nonlinear skewed distributions: \textbf{Log}, Box–Cox, or Yeo–Johnson transforms
% \item Tree-based models: usually no scaling required
% \end{itemize}
%
% \section*{5. “Shift and Scale” vs. “Zero-Mean Unit-Variance”}
%
% “Shift and scale” is a general affine transform:
% \[
% x' = a x + b
% \]
% while “zero-mean unit-variance” is a specific instance where
% \[
% a = \frac{1}{\sigma}, \quad b = -\frac{\mu}{\sigma}.
% \]
% Other choices of $(a,b)$ yield min–max or mean normalization.
% Thus, “shift and scale” is the umbrella; Z-score is a particular case.
%
% \section*{6. Summary}
% Normalization is model- and distribution-dependent.
% Standardization (Z-score) remains the most common and stable default for numerical continuous features when training models that depend on feature magnitudes or gradients.
%
% \end{document}
%

























\documentclass[11pt]{article}
\usepackage{amsmath, amssymb, booktabs, geometry, xcolor, graphicx}
\geometry{margin=1in}
\usepackage{array}
\usepackage{titlesec}
\usepackage{caption}
\captionsetup[table]{skip=6pt}
\usepackage{hyperref}

\title{\textbf{Normalization in Machine Learning: When to Use What}}
\author{}
\date{}

\begin{document}
\maketitle

\section*{1. Overview}
Normalization (or standardization/scaling) ensures features are comparable in scale and distribution.
It directly affects gradient-based optimization, distance-based metrics, and regularization.  
Different techniques suit different models and data distributions.

\begin{center}
\includegraphics[width=0.9\textwidth]{When_to_Use_What_.png}
\end{center}

\textit{(Conceptual illustration: raw features on different scales $\rightarrow$ normalized comparable scales.)}

\section*{2. Common Normalizations}

\begin{table}[h!]
\centering
\small
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{p{3.2cm} p{3cm} p{3.3cm} p{5cm}}
\toprule
\textbf{Name} & \textbf{Formula} & \textbf{Typical Use} & \textbf{Notes} \\
\midrule
\textbf{Zero-mean unit-variance (Z-score)} & $x' = \frac{x - \mu}{\sigma}$ & Linear models, SVM, PCA, neural nets & Works well when data $\sim$ Gaussian; sensitive to outliers. \\
\textbf{Min-max scaling} & $x' = \frac{x - x_{\min}}{x_{\max}-x_{\min}}$ & Distance-based models, bounded features & Preserves shape but can distort with outliers. \\
\textbf{Mean normalization} & $x' = \frac{x - \mu}{x_{\max}-x_{\min}}$ & Similar to above & Centers around 0; keeps shape. \\
\textbf{MaxAbs scaling} & $x' = \frac{x}{|x|_{\max}}$ & Sparse data (TF-IDF, embeddings) & Keeps sparsity pattern intact. \\
\textbf{Robust scaling} & $x' = \frac{x - \text{median}(x)}{\text{IQR}(x)}$ & Heavy-tailed or skewed data & Uses median and interquartile range. \\
\textbf{L2 normalization} & $\mathbf{x}' = \frac{\mathbf{x}}{\|\mathbf{x}\|_2}$ & Text or embedding vectors & Ensures unit-length vectors for cosine similarity. \\
\textbf{Log / Power transform} & $x' = \log(x + c)$ & Exponential or skewed features & Reduces skewness; often combined with standardization. \\
\textbf{Quantile / Rank transform} & $\text{Quantile} \mapsto \mathcal{N}(0,1)$ & Tree-based, ensemble, or robust pipelines & Equalizes marginal distributions. \\
\bottomrule
\end{tabular}
\caption{Common normalization and transformation methods.}
\end{table}

\section*{3. Model-Specific Guidelines}

\begin{table}[h!]
\centering
\small
\renewcommand{\arraystretch}{1.3}
\begin{tabular}{p{4cm} p{4cm} p{6cm}}
\toprule
\textbf{Model Type} & \textbf{Recommended Normalization} & \textbf{Reason} \\
\midrule
Linear regression, logistic regression, SVM & Z-score standardization & Ensures balanced gradient steps and regularization strength. \\
Neural networks & Z-score or min-max scaling & Keeps activations within non-saturated ranges; improves stability. \\
Tree-based models (RF, XGBoost, LightGBM) & None usually needed & Tree splits are order-based and scale-invariant. \\
k-NN, k-means, PCA, distance-based models & Z-score or min-max & Distance metrics depend on feature magnitudes. \\
Text / sparse vectors & MaxAbsScaler or L2 normalization & Preserves sparsity and vector norms. \\
Probabilistic models (Naive Bayes, count models) & None or log-transform & Assumes counts or frequencies directly. \\
\bottomrule
\end{tabular}
\caption{Normalization preferences by model family.}
\end{table}

\section*{4. Practical Heuristics}

\begin{itemize}
\item Different feature units (e.g., age, income): \textbf{StandardScaler}
\item Extreme outliers: \textbf{RobustScaler} or log-transform + StandardScaler
\item Features bounded in $[0,1]$: \textbf{MinMaxScaler}
\item Sparse text data: \textbf{MaxAbsScaler} or \textbf{L2 normalization}
\item Nonlinear skewed distributions: \textbf{Log}, Box–Cox, or Yeo–Johnson transforms
\item Tree-based models: usually no scaling required
\end{itemize}

\section*{5. “Shift and Scale” vs. “Zero-Mean Unit-Variance”}

“Shift and scale” is a general affine transform:
\[
x' = a x + b
\]
while “zero-mean unit-variance” is a specific instance where
\[
a = \frac{1}{\sigma}, \quad b = -\frac{\mu}{\sigma}.
\]
Other choices of $(a,b)$ yield min–max or mean normalization.  
Thus, “shift and scale” is the umbrella; Z-score is a particular case.

\section*{6. Dataset- and Task-Specific Normalization (Academic ML)}

\textbf{Vision:}  
For datasets like CIFAR-10, ImageNet, MNIST:  
pixel values $\in [0,255]$ are divided by $255$, sometimes followed by channel-wise normalization using ImageNet statistics:
\[
x' = \frac{x - \mu_{\text{channel}}}{\sigma_{\text{channel}}}.
\]

\textbf{NLP:}  
For embeddings (word2vec, BERT, etc.), apply \textbf{L2 normalization} to ensure cosine similarity is meaningful.  
For TF-IDF or bag-of-words features, use \textbf{MaxAbsScaler} or normalize rows to unit norm.

\textbf{Audio / Speech:}  
Log-mel spectrograms are often log-transformed and standardized per frequency bin.  
\textbf{Cepstral Mean and Variance Normalization (CMVN)} is common in speech recognition.

\textbf{Tabular / UCI-style datasets:}  
Use \textbf{Z-score} scaling for continuous variables, one-hot encode categorical, and optionally apply \textbf{RobustScaler} for outliers (e.g., Adult, Housing, Credit).

\textbf{Time Series:}  
Normalize per series using running mean/std or z-score per window; min-max scaling for bounded signals.

\section*{7. Industry-Scale Data Normalization}

In production ML systems (e.g., finance, ad-tech, e-commerce):
\begin{itemize}
\item \textbf{Streaming normalization:} statistics computed online via exponential moving averages.
\item \textbf{Feature Store pipelines:} maintain global $\mu$ and $\sigma$ per feature ID.
\item \textbf{Log transforms:} revenue, counts, and time deltas are log-transformed for stability.
\item \textbf{Winsorization:} clip extreme outliers (e.g., 1st and 99th percentile).
\item \textbf{Quantile normalization:} map heavy-tailed metrics to uniform or normal marginals.
\item \textbf{Feature hashing and embedding scaling:} in large-scale categorical features, embeddings are often normalized to unit variance.
\item \textbf{Whitening transformations:} applied before PCA or for anomaly detection.
\end{itemize}

Typical pipeline example (ad ranking or recommendation systems):
\[
\text{Log-scale counts} \rightarrow \text{clip outliers} \rightarrow \text{Z-score normalization}.
\]

\section*{8. Deep Neural Network–Specific Normalization Techniques}

Beyond input normalization, modern deep networks use internal normalization layers:

\begin{itemize}
\item \textbf{Batch Normalization (BN)}: normalizes activations per batch and feature:
\[
\hat{x}_i = \frac{x_i - \mu_{\mathcal{B}}}{\sqrt{\sigma_{\mathcal{B}}^2 + \epsilon}}, \quad y_i = \gamma \hat{x}_i + \beta
\]
BN stabilizes training, allows higher learning rates, and regularizes the model.

\item \textbf{Layer Normalization (LN)}: normalizes across features for each sample (used in Transformers):
\[
\hat{x} = \frac{x - \text{mean}(x)}{\text{std}(x)}.
\]

\item \textbf{Instance Normalization (IN)}: per-sample and per-channel normalization (style transfer, vision).

\item \textbf{Group Normalization (GN)}: hybrid of BN and LN; divides channels into groups and normalizes within each.

\item \textbf{Weight Normalization and Spectral Normalization:}  
Reparameterize weights to stabilize optimization or constrain Lipschitz constant.

\end{itemize}

\textbf{Input normalization remains crucial} even with internal BN or LN; data centered near zero and with unit variance accelerates convergence and prevents saturation in early layers.

\section*{9. Summary}
Normalization is model- and distribution-dependent.  
Standardization (Z-score) remains the most common and stable default for numerical continuous features when training models that depend on feature magnitudes or gradients.  
Deep networks further rely on internal normalization layers for stability.

\end{document}

