<!DOCTYPE html>

<html xmlns="http://www.w3.org/1999/xhtml">

<head>

<meta charset="utf-8" />
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta name="generator" content="pandoc" />


<meta name="author" content="Lei Sun" />

<meta name="date" content="2017-06-14" />

<title>Posterior Inference with Gaussian Derivative Likelihood: Model and Result</title>

<script src="site_libs/jquery-1.11.3/jquery.min.js"></script>
<meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="site_libs/bootstrap-3.3.5/css/cosmo.min.css" rel="stylesheet" />
<script src="site_libs/bootstrap-3.3.5/js/bootstrap.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/html5shiv.min.js"></script>
<script src="site_libs/bootstrap-3.3.5/shim/respond.min.js"></script>
<script src="site_libs/jqueryui-1.11.4/jquery-ui.min.js"></script>
<link href="site_libs/tocify-1.9.1/jquery.tocify.css" rel="stylesheet" />
<script src="site_libs/tocify-1.9.1/jquery.tocify.js"></script>
<script src="site_libs/navigation-1.1/tabsets.js"></script>
<link href="site_libs/highlightjs-1.1/textmate.css" rel="stylesheet" />
<script src="site_libs/highlightjs-1.1/highlight.js"></script>
<link href="site_libs/font-awesome-4.5.0/css/font-awesome.min.css" rel="stylesheet" />

<style type="text/css">code{white-space: pre;}</style>
<style type="text/css">
  pre:not([class]) {
    background-color: white;
  }
</style>
<script type="text/javascript">
if (window.hljs && document.readyState && document.readyState === "complete") {
   window.setTimeout(function() {
      hljs.initHighlighting();
   }, 0);
}
</script>



<style type="text/css">
h1 {
  font-size: 34px;
}
h1.title {
  font-size: 38px;
}
h2 {
  font-size: 30px;
}
h3 {
  font-size: 24px;
}
h4 {
  font-size: 18px;
}
h5 {
  font-size: 16px;
}
h6 {
  font-size: 12px;
}
.table th:not([align]) {
  text-align: left;
}
</style>


</head>

<body>

<style type = "text/css">
.main-container {
  max-width: 940px;
  margin-left: auto;
  margin-right: auto;
}
code {
  color: inherit;
  background-color: rgba(0, 0, 0, 0.04);
}
img {
  max-width:100%;
  height: auto;
}
.tabbed-pane {
  padding-top: 12px;
}
button.code-folding-btn:focus {
  outline: none;
}
</style>


<style type="text/css">
/* padding for bootstrap navbar */
body {
  padding-top: 51px;
  padding-bottom: 40px;
}
/* offset scroll position for anchor links (for fixed navbar)  */
.section h1 {
  padding-top: 56px;
  margin-top: -56px;
}

.section h2 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h3 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h4 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h5 {
  padding-top: 56px;
  margin-top: -56px;
}
.section h6 {
  padding-top: 56px;
  margin-top: -56px;
}
</style>

<script>
// manage active state of menu based on current page
$(document).ready(function () {
  // active menu anchor
  href = window.location.pathname
  href = href.substr(href.lastIndexOf('/') + 1)
  if (href === "")
    href = "index.html";
  var menuAnchor = $('a[href="' + href + '"]');

  // mark it active
  menuAnchor.parent().addClass('active');

  // if it's got a parent navbar menu mark it active as well
  menuAnchor.closest('li.dropdown').addClass('active');
});
</script>


<div class="container-fluid main-container">

<!-- tabsets -->
<script>
$(document).ready(function () {
  window.buildTabsets("TOC");
});
</script>

<!-- code folding -->




<script>
$(document).ready(function ()  {

    // move toc-ignore selectors from section div to header
    $('div.section.toc-ignore')
        .removeClass('toc-ignore')
        .children('h1,h2,h3,h4,h5').addClass('toc-ignore');

    // establish options
    var options = {
      selectors: "h1,h2,h3",
      theme: "bootstrap3",
      context: '.toc-content',
      hashGenerator: function (text) {
        return text.replace(/[.\\/?&!#<>]/g, '').replace(/\s/g, '_').toLowerCase();
      },
      ignoreSelector: ".toc-ignore",
      scrollTo: 0
    };
    options.showAndHide = true;
    options.smoothScroll = true;

    // tocify
    var toc = $("#TOC").tocify(options).data("toc-tocify");
});
</script>

<style type="text/css">

#TOC {
  margin: 25px 0px 20px 0px;
}
@media (max-width: 768px) {
#TOC {
  position: relative;
  width: 100%;
}
}


.toc-content {
  padding-left: 30px;
  padding-right: 40px;
}

div.main-container {
  max-width: 1200px;
}

div.tocify {
  width: 20%;
  max-width: 260px;
  max-height: 85%;
}

@media (min-width: 768px) and (max-width: 991px) {
  div.tocify {
    width: 25%;
  }
}

@media (max-width: 767px) {
  div.tocify {
    width: 100%;
    max-width: none;
  }
}

.tocify ul, .tocify li {
  line-height: 20px;
}

.tocify-subheader .tocify-item {
  font-size: 0.90em;
  padding-left: 25px;
  text-indent: 0;
}

.tocify .list-group-item {
  border-radius: 0px;
}


</style>

<!-- setup 3col/9col grid for toc_float and main content  -->
<div class="row-fluid">
<div class="col-xs-12 col-sm-4 col-md-3">
<div id="TOC" class="tocify">
</div>
</div>

<div class="toc-content col-xs-12 col-sm-8 col-md-9">




<div class="navbar navbar-default  navbar-fixed-top" role="navigation">
  <div class="container">
    <div class="navbar-header">
      <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar">
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
        <span class="icon-bar"></span>
      </button>
      <a class="navbar-brand" href="index.html">truncash</a>
    </div>
    <div id="navbar" class="navbar-collapse collapse">
      <ul class="nav navbar-nav">
        <li>
  <a href="index.html">Home</a>
</li>
<li>
  <a href="about.html">About</a>
</li>
<li>
  <a href="license.html">License</a>
</li>
      </ul>
      <ul class="nav navbar-nav navbar-right">
        <li>
  <a href="https://github.com/LSun/truncash">
    <span class="fa fa-github"></span>
     
  </a>
</li>
      </ul>
    </div><!--/.nav-collapse -->
  </div><!--/.container -->
</div><!--/.navbar -->

<div class="fluid-row" id="header">



<h1 class="title toc-ignore">Posterior Inference with Gaussian Derivative Likelihood: Model and Result</h1>
<h4 class="author"><em>Lei Sun</em></h4>
<h4 class="date"><em>2017-06-14</em></h4>

</div>


<!-- The file analysis/chunks.R contains chunks that define default settings
shared across the workflowr files. -->
<!-- Update knitr chunk options -->
<!-- Insert the date the file was last updated -->
<p><strong>Last updated:</strong> 2017-06-17</p>
<!-- Insert the code version (Git commit SHA1) if Git repository exists and R
 package git2r is installed -->
<p><strong>Code version:</strong> 73b4bb7</p>
<!-- Add your analysis here -->
<div id="gd-ash-model" class="section level2">
<h2><code>GD-ASH</code> Model</h2>
<p>Recall the typical <code>GD-ASH</code> model is</p>
<p><span class="math display">\[
\begin{array}{l}
\beta_j \sim \sum\pi_k N\left(0, \sigma_k^2\right) \ ;\\
\hat\beta_j = \beta_j + \hat s_j z_j  \ ;\\
z_j \sim N\left(0, 1\right), \text{ correlated} \ .
\end{array}
\]</span> Then we are fitting the empirical distribution of <span class="math inline">\(z\)</span> with Gaussian derivatives</p>
<p><span class="math display">\[
f(z) = \sum w_l\frac{1}{\sqrt{l!}}\varphi^{(l)}(z) \ .
\]</span> Therefore, in essence, we are changing the likelihood of <span class="math inline">\(\hat\beta_j | \hat s_j, \beta_j\)</span> from correlated <span class="math inline">\(N\left(\beta_j, \hat s_j^2\right)\)</span> to independent <span class="math inline">\(\frac{1}{\hat s_j}f\left(\frac{\hat\beta_j - \beta_j}{\hat s_j}\right)\)</span>, which using Gaussian derivatives is</p>
<p><span class="math display">\[
\frac{1}{\hat s_j}\sum w_l \frac{1}{\sqrt{l!}} \varphi^{(l)}\left(\frac{\hat\beta_j - \beta_j}{\hat s_j}\right) \ .
\]</span> Note that when <span class="math inline">\(f = \varphi\)</span> it becomes the independent <span class="math inline">\(N\left(\beta_j, \hat s_j^2\right)\)</span> case.</p>
</div>
<div id="gd-lik-model" class="section level2">
<h2><code>GD-Lik</code> Model</h2>
<p>Therefore, if we use Gaussian derivatives instead of Gaussian as the likelihood, the posterior distribution of <span class="math inline">\(\beta_j | \hat s_j, \hat\beta_j\)</span> should be</p>
<p><span class="math display">\[
\begin{array}{rcl}
f\left(\beta_j \mid \hat s_j, \hat\beta_j\right)
&amp;=&amp;
\frac{
\displaystyle g\left(\beta_j\right)
f\left(\hat\beta_j \mid \hat s_j, \beta_j \right)
}{
\displaystyle\int g\left(\beta_j\right) f\left(\hat\beta_j \mid \hat s_j, \beta_j \right) d\beta_j
}\\
&amp;=&amp;
\frac{
\displaystyle
\sum\pi_k\sum w_l 
\frac{1}{\sigma_k}
\varphi\left(\frac{\beta_j - \mu_k}{\sigma_k}\right)
\frac{1}{\hat s_j}
\frac{1}{\sqrt{l!}}
\varphi^{(l)}\left(\frac{\hat\beta_j - \beta_j}{\hat s_j}\right)
}{
\displaystyle
\sum\pi_k\sum w_l \int
\frac{1}{\sigma_k}
\varphi\left(\frac{\beta_j - \mu_k}{\sigma_k}\right)
\frac{1}{\hat s_j}
\frac{1}{\sqrt{l!}}
\varphi^{(l)}\left(\frac{\hat\beta_j - \beta_j}{\hat s_j}\right)
d\beta_j
} \ .
\end{array}
\]</span> The denominator <a href="ash_gd.html#normal_mixture_prior">readily has an analytic form</a> which is</p>
<p><span class="math display">\[
\displaystyle
\sum\pi_k \sum w_l
\frac{\hat s_j^l}{\left(\sqrt{\sigma_k^2 + \hat s_j^2}\right)^{l+1}}
\frac{1}{\sqrt{l!}}
\varphi^{(l)}\left(\frac{\hat\beta_j - \mu_k}{\sqrt{\sigma_k^2 + \hat s_j^2}}\right)
:=
\sum \pi_k \sum w_l f_{jkl}
\ .
\]</span></p>
</div>
<div id="posterior-mean" class="section level2">
<h2>Posterior mean</h2>
<p>After algebra, the posterior mean is given by</p>
<p><span class="math display">\[
E\left[\beta_j \mid \hat s_j, \hat \beta_j \right]
=
\int \beta_j
f\left(\beta_j \mid \hat s_j, \hat\beta_j\right) d\beta_j
=
\displaystyle
\frac{
\sum \pi_k \sum w_l m_{jkl}
}{
\sum \pi_k \sum w_l f_{jkl}
} \ ,
\]</span> where <span class="math inline">\(f_{jkl}\)</span> is defined as above and <span class="math display">\[
m_{jkl} =
-
\frac{\hat s_j^l \sigma_k^2}{\left(\sqrt{\sigma_k^2 + \hat s_j^2}\right)^{l+2}}
\frac{1}{\sqrt{l!}}
\varphi^{(l+1)}\left(\frac{\hat\beta_j - \mu_k}{\sqrt{\sigma_k^2 + \hat s_j^2}}\right)
+
\frac{\hat s_j^l\mu_k}{\left(\sqrt{\sigma_k^2 + \hat s_j^2}\right)^{l+1}}
\frac{1}{\sqrt{l!}}
\varphi^{(l)}\left(\frac{\hat\beta_j - \mu_k}{\sqrt{\sigma_k^2 + \hat s_j^2}}\right) \ .
\]</span></p>
</div>
<div id="local-fdr" class="section level2">
<h2>Local FDR</h2>
<p>Assuming <span class="math inline">\(\mu_k \equiv 0\)</span>, the <code>lfdr</code> is given by <span class="math display">\[
p\left(\beta_j = 0\mid \hat s_j, \hat \beta_j\right)
=
\frac{
\pi_0
\sum w_l 
\frac{1}{\hat s_j}
\frac{1}{\sqrt{l!}}
\varphi^{(l)}\left(\frac{\hat\beta_j}{\hat s_j}\right)
}{
\sum \pi_k
\sum w_l
f_{jkl}
} \ .
\]</span></p>
</div>
<div id="local-fsr" class="section level2">
<h2>Local FSR</h2>
<p>Right now the analytic form of <code>lfsr</code> using Gaussian derivatives is unavailable.</p>
</div>
<div id="simulation" class="section level2">
<h2>Simulation</h2>
<p>The correlated <span class="math inline">\(N\left(0, 1\right)\)</span> <span class="math inline">\(z\)</span> scores are simulated from the GTEx/Liver data by the <a href="nullpipeline.html">null pipeline</a>. In order to get a better sense of the effectiveness of <code>GD-ASH</code> and <code>GD-Lik</code>, we are using data sets more distorted by correlation in the simulation. In particular, we are using an “inflation” batch, defined as the standard error of the correlated <span class="math inline">\(z\)</span> no less than <span class="math inline">\(1.2\)</span>, and a “deflation” batch, defined as that no greater than <span class="math inline">\(0.8\)</span>. Out of <span class="math inline">\(1000\)</span> simulated data sets, there are <span class="math inline">\(109\)</span> inflation ones and <span class="math inline">\(99\)</span> deflation ones.</p>
<p>In order to create realistic heterskedastic estimated standard error, <span class="math inline">\(\hat s_j\)</span>’s are also simulated from the same <a href="nullpipeline.html">null pipeline</a>. Let <span class="math inline">\(\sigma^2 = \frac1n \sum\limits_{j = 1}^n \hat s_j^2\)</span> be the average strength of the heteroskedastic noise, and the true effects <span class="math inline">\(\beta_j\)</span>’s are simulated from <span class="math display">\[
0.6\delta_0 + 0.3N\left(0, \sigma^2\right) + 0.1N\left(0, \left(2\sigma\right)^2\right) \ .
\]</span></p>
<p>Then let <span class="math inline">\(\hat\beta_j = \beta_j + \hat s_j z_j\)</span>. We are using <span class="math inline">\(\hat\beta_j\)</span>, <span class="math inline">\(\hat s_j\)</span>, along with <span class="math inline">\(\hat z_j = \hat\beta_j / \hat s_j\)</span>, <span class="math inline">\(\hat p_j= 2\left(1 - \Phi\left(\left|\hat z_j\right|\right)\right)\)</span>, as the summary statistics fed to <code>GD-ASH</code> and <code>GD-Lik</code>, as well as into <code>BH</code>, <code>qvalue</code>, <code>locfdr</code>, <code>ASH</code> for a comparison.</p>
<pre class="r"><code>source(&quot;../code/gdash_lik.R&quot;)</code></pre>
<pre class="r"><code>z.mat = readRDS(&quot;../output/z_null_liver_777.rds&quot;)
se.mat = readRDS(&quot;../output/sebetahat_null_liver_777.rds&quot;)</code></pre>
<pre class="r"><code>z.sd = apply(z.mat, 1, sd)
inflation.index = (z.sd &gt;= 1.2)
deflation.index = (z.sd &lt;= 0.8)
z.inflation = z.mat[inflation.index, ]
se.inflation = se.mat[inflation.index, ]
## Number of inflation data sets
nrow(z.inflation)</code></pre>
<pre><code>[1] 109</code></pre>
<pre class="r"><code>z.deflation = z.mat[deflation.index, ]
se.deflation = se.mat[deflation.index, ]
## Number of deflation data sets
nrow(z.deflation)</code></pre>
<pre><code>[1] 99</code></pre>
</div>
<div id="inflation-data-sets" class="section level2">
<h2>Inflation data sets</h2>
<div id="some-examples-of-inflated-correlated-null-z-scores" class="section level3">
<h3>Some examples of inflated correlated null <span class="math inline">\(z\)</span> scores</h3>
<p><img src="figure/gd_lik_2.rmd/inflation%20examples-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="hat-pi_0" class="section level3">
<h3><span class="math inline">\(\hat \pi_0\)</span></h3>
<p><code>locfdr</code> overestimates, <code>ASH</code> underestimates, <code>GD-ASH</code> on target, <code>qvalue</code> surprisingly good.</p>
<p><img src="figure/gd_lik_2.rmd/inflation%20pihat0-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="mse" class="section level3">
<h3>MSE</h3>
<p><code>GD-Lik</code> clearly improves the posterior estimates of <code>GD-ASH</code>.</p>
<p><img src="figure/gd_lik_2.rmd/inflation%20mse-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="auc" class="section level3">
<h3>AUC</h3>
<p>For almost all methods, <code>.q</code> means using q values, and <code>.l</code> using local FDRs. <code>GD-Lik</code> is the best, yet even the vanilla <span class="math inline">\(p\)</span> values are not much worse. It indicates that all the methods based on summary statistics indeed make few changes to the order of original <span class="math inline">\(p\)</span> values. Worth noting is that <code>locfdr</code> doesn’t perform well, and <code>lfdr</code>’s give a drastically different result than q values do, probably due to some artifacts like <a href="auc_pvalue.html">ties</a>.</p>
<p><img src="figure/gd_lik_2.rmd/inflation%20auc-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="q-values-and-positive-fdr-pfdr-calibration" class="section level3">
<h3>q values and positive FDR (pFDR) calibration</h3>
<p>Dashed lines are <span class="math inline">\(y = x\)</span> and <span class="math inline">\(y = 2x\)</span>. <code>ASH</code> and <code>qvalue</code> are too liberal, and <code>locfdr</code> is too conservative. <code>GD-ASH</code> and <code>BH</code> give very similar results and not far off. <code>BH</code>’s calibrates pFDR relatively well, even though it’s only guaranteed to control FDR under independence. <code>GD-Lik</code> calibrates pFDR almost precisely.</p>
<p><img src="figure/gd_lik_2.rmd/inflation%20qvalue%20calibration%20plotting-1.png" width="672" style="display: block; margin: auto;" /><img src="figure/gd_lik_2.rmd/inflation%20qvalue%20calibration%20plotting-2.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="s-values-and-positive-fsr-pfsr-calibration" class="section level3">
<h3>s values and positive FSR (pFSR) calibration</h3>
<p>Both <code>ASH</code> and <code>GD-ASH</code> are too liberal, although <code>GD-ASH</code> is not too far off.</p>
<p><img src="figure/gd_lik_2.rmd/inflation%20svalue%20calibration%20plotting-1.png" width="672" style="display: block; margin: auto;" /><img src="figure/gd_lik_2.rmd/inflation%20svalue%20calibration%20plotting-2.png" width="672" style="display: block; margin: auto;" /></p>
</div>
</div>
<div id="deflation-data-sets" class="section level2">
<h2>Deflation data sets</h2>
<div id="some-examples-of-deflated-correlated-null-z-scores" class="section level3">
<h3>Some examples of deflated correlated null <span class="math inline">\(z\)</span> scores</h3>
<p><img src="figure/gd_lik_2.rmd/deflation%20examples-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="hat-pi_0-1" class="section level3">
<h3><span class="math inline">\(\hat \pi_0\)</span></h3>
<p>Almost all methods except <code>GD-ASH</code> overestimate as expected. <code>GD-ASH</code> occasionally severely underestimates as <a href="simulation_real_se.html#global_null">seen before</a>.</p>
<p><img src="figure/gd_lik_2.rmd/deflation%20pihat0-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="mse-1" class="section level3">
<h3>MSE</h3>
<p><code>GD-Lik</code> does better than <code>ASH</code> and <code>GD-ASH</code> but not as significantly as in the inflation case.</p>
<p><img src="figure/gd_lik_2.rmd/deflation%20mse-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="auc-1" class="section level3">
<h3>AUC</h3>
<p>Similar story as in the inflation case, although this time <code>qvalue</code>’s <code>lfdr</code> behaves weirdly.</p>
<p><img src="figure/gd_lik_2.rmd/deflation%20auc-1.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="q-values-and-positive-fdr-pfdr-calibration-1" class="section level3">
<h3>q values and positive FDR (pFDR) calibration</h3>
<p>Essentially all methods successfully control pFDR. <code>GD-Lik</code> looks good although off a little. <code>qvalue</code> is the most conservative, followed by <code>ASH</code>, <code>GD-ASH</code>, and <code>locfdr</code>.</p>
<p><img src="figure/gd_lik_2.rmd/deflation%20qvalue%20calibration%20plotting-1.png" width="672" style="display: block; margin: auto;" /><img src="figure/gd_lik_2.rmd/deflation%20qvalue%20calibration%20plotting-2.png" width="672" style="display: block; margin: auto;" /></p>
</div>
<div id="s-values-and-positive-fsr-pfsr-calibration-1" class="section level3">
<h3>s values and positive FSR (pFSR) calibration</h3>
<p>Both <code>ASH</code> and <code>GD-ASH</code> seem too conservative, although <code>GD-ASH</code> is more powerful.</p>
<p><img src="figure/gd_lik_2.rmd/deflation%20svalue%20calibration%20plotting-1.png" width="672" style="display: block; margin: auto;" /><img src="figure/gd_lik_2.rmd/deflation%20svalue%20calibration%20plotting-2.png" width="672" style="display: block; margin: auto;" /></p>
</div>
</div>
<div id="remarks" class="section level2">
<h2>Remarks</h2>
<ol style="list-style-type: decimal">
<li>Would be nice to come up with a way to calculate <code>lfsr</code> in <code>GD-Lik</code>.</li>
<li>Many methods are too liberal for inflation cases and too conservative for deflation cases, showing a lack of robustness against correlation. Although, on average they probably seem about right.</li>
</ol>
</div>
<div id="session-information" class="section level2">
<h2>Session information</h2>
<!-- Insert the session information into the document -->
<pre class="r"><code>sessionInfo()</code></pre>
<pre><code>R version 3.3.3 (2017-03-06)
Platform: x86_64-apple-darwin13.4.0 (64-bit)
Running under: macOS Sierra 10.12.5

locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
[1] ashr_2.1-19       Rmosek_7.1.3      PolynomF_0.94     cvxr_0.0.0.9009  
[5] REBayes_0.62      Matrix_1.2-8      SQUAREM_2016.10-1 EQL_1.0-0        
[9] ttutils_1.0-1    

loaded via a namespace (and not attached):
 [1] Rcpp_0.12.11      knitr_1.16        magrittr_1.5     
 [4] MASS_7.3-45       pscl_1.4.9        doParallel_1.0.10
 [7] lattice_0.20-34   foreach_1.4.3     stringr_1.2.0    
[10] tools_3.3.3       parallel_3.3.3    grid_3.3.3       
[13] git2r_0.18.0      iterators_1.0.8   htmltools_0.3.6  
[16] yaml_2.1.14       rprojroot_1.2     digest_0.6.12    
[19] codetools_0.2-15  evaluate_0.10     rmarkdown_1.5    
[22] stringi_1.1.2     backports_1.0.5   truncnorm_1.0-7  </code></pre>
</div>

<hr>
<p>
    This <a href="http://rmarkdown.rstudio.com">R Markdown</a> site was created with <a href="https://github.com/jdblischak/workflowr">workflowr</a>
</p>
<hr>

<!-- To enable disqus, uncomment the section below and provide your disqus_shortname -->

<!-- disqus
  <div id="disqus_thread"></div>
    <script type="text/javascript">
        /* * * CONFIGURATION VARIABLES: EDIT BEFORE PASTING INTO YOUR WEBPAGE * * */
        var disqus_shortname = 'rmarkdown'; // required: replace example with your forum shortname

        /* * * DON'T EDIT BELOW THIS LINE * * */
        (function() {
            var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
            dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
            (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
        })();
    </script>
    <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>
    <a href="http://disqus.com" class="dsq-brlink">comments powered by <span class="logo-disqus">Disqus</span></a>
-->


</div>
</div>

</div>

<script>

// add bootstrap table styles to pandoc tables
function bootstrapStylePandocTables() {
  $('tr.header').parent('thead').parent('table').addClass('table table-condensed');
}
$(document).ready(function () {
  bootstrapStylePandocTables();
});


</script>

<!-- dynamically load mathjax for compatibility with self-contained -->
<script>
  (function () {
    var script = document.createElement("script");
    script.type = "text/javascript";
    script.src  = "https://mathjax.rstudio.com/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML";
    document.getElementsByTagName("head")[0].appendChild(script);
  })();
</script>

</body>
</html>