end of hw 2

dm2.tex

@@ -145,11 +145,11 @@
 	\end{array} $$
 	We then compute the dual function :
 	$$ \begin{array}{lll}
-	g(\nu) & = & \inf_{x, y} \| y \|_2^2 - \nu^\trans y + \nu^\trans A x + \| x \|_1 - \nu^\trans b \\[2mm]
-	& = & \| \nu \|_2^{2 \, *} + \left\| - A^\trans \nu \right\|_1^* - \nu^\trans b \\[2mm]
+	g(\nu) & = & \inf_{x, y} \| y \|_2^2 - \nu^\trans y + \nu^\trans A x + \| x \|_1 + \nu^\trans b \\[2mm]
+	& = & - \| \nu \|_2^{2 \, *} - \left\| - A^\trans \nu \right\|_1^* + \nu^\trans b \\[2mm]
 	& = & \syst{ll}{
-		\| \nu \|_2^{2 \, *} - \nu^\trans b & \text{If } \, \left\| A^\trans \nu \right\|_{\infty} \leqslant 1 \\
-		+ \infty & \text{Otherwise}
+		\nu^\trans b - \| \nu \|_2^{2 \, *} & \text{If } \, \left\| A^\trans \nu \right\|_{\infty} \leqslant 1 \\
+		- \infty & \text{Otherwise}
 	}
 	\end{array} $$
 	Now we compute the conjugate of the square of the 2-norm :
@@ -164,7 +164,7 @@
 	Then our dual can be written :
 	\begin{center}
 		\fbox{$ \displaystyle \text{(Dual RLS)} \quad \begin{array}{llr}
-			\max_\nu & \frac{1}{4}\| \nu \|_2^2 - b^\trans \nu \\[2mm]
+			\max_\nu & b^\trans \nu - \frac{1}{4}\| \nu \|_2^2 \\[2mm]
 			\text{s.t.} & \left\| A^\trans \nu \right\|_{\infty} \leqslant 1
 			\end{array} $}
 	\end{center}
@@ -175,7 +175,7 @@
 	First, we can remark that (Sep. 2) is feasible by simply taking the point $(0, \mathbf{1})$ and bounded because the objective function is non-negative for $z \succeq 0$. \\
 	Let $(\omega, z)$ a feasible point. Suppose that there exists $i$ such that $z_i > \mathcal{L}(\omega, x_i, y_i)$. Then we let $z'$ such that $z'_j = z_j$ for $j \neq i$ and $z'_i = \mathcal{L}(\omega, x_i, y_i)$. We have in particular : \vspace{-2mm}
 	$$ z'_i \geqslant 0 \qquad z'_i \geqslant 1 - y_i (\omega^\trans x_i) $$
-	And the other constraints are also verified because $z$ is feasible. Because the factor of $z_i$ is strictly positive in the objective function and because $z'_i < z_i$, the value associated to the point $(\omega, z')$ is strictly inferior to the value of $(\omega, z)$. \\
+	Furthermore the other constraints are also verified because $z$ is feasible. Because the factor of $z_i$ is strictly positive in the objective function and because $z'_i < z_i$, the value associated to the point $(\omega, z')$ is strictly inferior to the value of $(\omega, z)$. \\
 	Then for $(\omega^*, z^*)$ optimal we have $z_i^* \leqslant \mathcal{L}(\omega^*, x_i, y_i)$ for every $i$. Furthermore, because of constraints, this inequality is in fact an equality. So : \vspace{-2mm}
 	$$ z_i^* = \mathcal{L}(\omega^*, x_i, y_i) $$
 	In this case the optimal value is :
@@ -222,7 +222,7 @@
 	\paragraph{}
 	We denote by $p^*(x)$ the value:
 	$$ p^*(x) = \sup_{a \in \mathcal{P}} a^\trans x = - \inf_{a \in \mathcal{P}} - a^\trans x $$
-	Then $p(x)$ is the opposite of the optimal value of a linear program. We know that strong duality holds in this case and we compute the dual function associated to (P) $\inf_{a \in \mathcal{P}} - a^\trans x$:
+	Then $p^*(x)$ is the opposite of the optimal value of a linear program. We know that strong duality holds in this case and we compute the dual function associated to (P) $\inf_{a \in \mathcal{P}} - a^\trans x$:
 	$$ g(a, z) = \inf_a -a^\trans x + z^\trans C^\trans a - z^\trans d = \syst{ll}{
 		-z^\trans d & \text{If } \, Cz = x \\
 		- \infty & \text{Otherwise}
@@ -250,7 +250,7 @@
 	We just proved the equivalence \eqref{equiv} that allows us to conclude :
 	\begin{center}
 		\fbox{$ \displaystyle \begin{array}{llr}
-			\max_x & c^\trans x \\
+			\min_x & c^\trans x \\
 			\text{s.t.} & \sup_{a \in \mathcal{P}} a^\trans x \leqslant b
 			\end{array} \qquad \Leftrightarrow \qquad \begin{array}{llr}
 			\min_{x, z} & c^\trans x \\
@@ -259,5 +259,69 @@
 			& z \succeq 0
 			\end{array} $}
 	\end{center}
+
+	\subsection*{Exercise 5 (Boolean LP)}
+	
+	\paragraph{1.}
+	We first compute the Lagrangian of our problem: \vspace{-3mm}
+	$$ L(x, \lambda, \nu) =  c^\trans x + \lambda^\trans (Ax - b) + \sum_{i = 1}^n \nu_i x_i (1 - x_i) \vspace{-2mm} $$
+	Now we compute the gradient with respect to $x_i$ of the Lagrangian:
+	$$ \nabla_{x_i} L(x, \lambda, \nu) = c_i + a_i^\trans \lambda + \nu_i (1 - 2 x_i) \qquad \text{where } \; A = \pmat{a_1 & \cdots & a_n} $$
+	This gradient depends only on $x_i$ then the minimum of $L$ w.r.t $x$ is obtained by optimizing on each $x_i$ independently. We can remark that if $\nu_i > 0$, then $L(\alpha e_i, \lambda, \nu)$ tends to $- \infty$ when $\alpha$ tends to $+\infty$. So, for $L$ to be bounded, we need $\nu \preceq 0$. In this case $L$ is convex in $x$ and we just need to find the vanishing point of the gradient to obtain the minimum of $L$:
+	$$ \nabla_{x_i} L(x, \lambda, \nu) = 0 \quad \Leftrightarrow \quad \left( \nu_i < 0 \, \text{ and } \, x_i = \dfrac{c_i + a_i^\trans \lambda + \nu_i}{2 \nu_i} \right) \; \text{or} \; \left( \nu_i = 0 \, \text{ and } \, c_i + a_i^\trans \lambda = 0 \right) $$
+	Now we write:
+	$$ L(x, \lambda, \nu) = - \lambda^\trans b + \sum_{i = 1}^n L_i(x_i, \lambda, \nu_i) \qquad \text{where } \; L_i(x_i, \lambda, \nu_i) = \left( c_i + a_i^\trans \lambda + \nu_i - \nu_i x_i \right) x_i $$
+	Such that the dual function wan be written:
+	$$ g(\lambda, \nu) = \inf_x L(x, \lambda, \nu) = - \lambda^\trans b + \sum_{i = 1}^n \inf_{x_i} L_i(x_i, \lambda, \nu_i) = -\lambda^\trans b + \sum_{i = 1}^n g_i(\lambda, \nu_i) $$
+	With the following expression of $g_i$, from what we have done before:
+	$$ g_i(\lambda, \nu_i) = \syst{ll}{
+		L_i \left( \dfrac{c_i + a_i^\trans \lambda + \nu_i}{2 \nu_i}, \, \lambda, \, \nu_i \right) & \text{If } \; \nu_i < 0 \\
+		0 & \text{If } \; \nu_i = 0 \; \text{ and } \; c_i + a_i^\trans \lambda = 0 \\
+		- \infty & \text{Otherwise}
+	} $$
+	We simplify the case $\nu_i < 0$: \vspace{-3mm}
+	$$ \nu_i < 0 \: \Rightarrow \; g_i(\lambda, \nu_i) = \dfrac{1}{4 \nu_i} \left( c_i + a_i^\trans \lambda + \nu_i \right)^2 $$
+	Because $g$ depends on $\nu_i$ only in the term $g_i(\lambda, \nu_i)$ we can simplify the expression of the Lagrange dual by maximizing each $g_i$ with respect to $\nu_i$ and removing each $\nu_i$ from the dual variables. \\
+	From the hint given in the question we have:
+	$$ \sup_{\nu_i < 0} g_i(\lambda, \nu_i) = \min \left\{ 0, \, c_i + a_i^\trans \lambda \right\} $$
+	Furthermore :
+	$$ c_i + a_i^\trans \lambda = 0 \quad \Rightarrow \quad g_i(\lambda, 0) = 0 = \min \left\{ 0, \, c_i + a_i^\trans \lambda \right\} $$
+	Now we can remove the variable $\nu_i$ to obtain:
+	$$ g_i(\lambda) = \sup_{\nu_i} g_i(\lambda, \nu_i) = \min \left\{ 0, \, c_i + a_i^\trans \lambda \right\} $$
+	Finally we obtain the following expression for the Lagrange dual:
+	\begin{center}
+		\fbox{$ \displaystyle g(\lambda) = -b^\trans \lambda + \sum_{i = 1}^n \min \left\{ 0, \, c_i + a_i^\trans \lambda \right\} $}
+	\end{center}
+
+	\paragraph{2.}
+	We compute the Lagrange dual of the LP relaxation:
+	$$ h(x, \alpha, \beta, \gamma) = \inf_x \; c^\trans x + \alpha^\trans Ax - \alpha^\trans b - \beta^\trans x + \gamma^\trans x - \gamma^\trans \mathbf{1} = \syst{ll}{
+		- \alpha^\trans b - \gamma^\trans \mathbf{1} & \text{If } \; c + A^\trans \alpha + \gamma = \beta \\
+		- \infty & \text{Otherwise}
+	} $$
+	This gives the follwing dual problem:
+	$$ \begin{array}{llr}
+	\max_{\alpha, \beta, \gamma} & - \alpha^\trans b - \gamma^\trans \mathbf{1} \\
+	\text{s.t.} & c + A^\trans \alpha + \gamma = \beta \\
+	& \alpha, \beta, \gamma \succeq 0
+	\end{array} $$
+	We can inject the non-negativity of $\beta$ in the equality constraint and remove this variable:
+	$$ \begin{array}{llr}
+	\max_{\alpha, \gamma} & - \alpha^\trans b - \gamma^\trans \mathbf{1} \\
+	\text{s.t.} & c + A^\trans \alpha + \gamma \succeq 0 \\
+	& \alpha, \gamma \succeq 0
+	\end{array} $$
+	We also remark that the objective function is strictly decreasing with respect to $\gamma$. Then the optimal is obtain for $\gamma$ as small as possible when $\alpha$ is fixed. This leads us to:
+	$$ \gamma_i  = \max \left\{ 0, \, - c_i - a_i^\trans \alpha \right\} = - \min \left\{ 0, \, c_i + a_i^\trans \alpha \right\} $$
+	Now we are able to simplify again our dual:
+	$$ \text{(D2)} \quad \begin{array}{llr}
+	\max_{\alpha} & g(\alpha) = - \alpha^\trans b + \sum_{i = 1}^n \min \left\{ 0, \, c_i + a_i^\trans \alpha \right\} \\
+	\text{s.t.} & \alpha \succeq 0
+	\end{array} $$
+	We recognize the dual obtain in the previous question for the Lagrange relaxation.
+	\begin{center}
+		\fbox{The lower bound obtained via Lagrangian relaxation, and via the LP relaxation (2), are
+			the same.}
+	\end{center}
 	
 \end{document}
\ No newline at end of file