Jekyll2021-01-23T02:45:16+00:00/feed.xmlmj’s codeUse MacVim in Unity3d2021-01-22T12:05:00+00:002021-01-22T12:05:00+00:00/unity3d/2021/01/22/use-macvim-in-unity3d<h2 id="compile-omnisharp-roslyn-server">Compile omnisharp-roslyn server</h2>
<p>Clone <a href="https://github.com/OmniSharp/omnisharp-roslyn">omnisharp-roslyn</a></p>
<p>Use command <code class="language-plaintext highlighter-rouge">./build.sh</code> to compile omnisharp-roslyn server (omnisharp-roslyn will be compiled with local environment mono)</p>
<h2 id="vimrc-settings">.vimrc settings</h2>
<p>Add <code class="language-plaintext highlighter-rouge">Plug 'prabirshrestha/asyncomplete.vim'</code> and <code class="language-plaintext highlighter-rouge">Plug 'OmniSharp/omnisharp-vim'</code> in .vimrc and do :PlugInstall</p>
<p>Copy the following script to .vimrc</p>
<p>Don’t forget to change the value of <code class="language-plaintext highlighter-rouge">let g:OmniSharp_server_path = ''</code> to the absolute path of OmniSharp.exe which built just now.</p>
<figure class="highlight"><pre><code class="language-vimscript" data-lang="vimscript"><span class="c">" OmniSharp</span>
<span class="c">"" Compile omnisharp-roslyn locally and set the artifacts OmniSharp.exe to OmniSharp_server_path</span>
<span class="c">"" https://github.com/OmniSharp/omnisharp-roslyn</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_server_path</span> <span class="p">=</span> <span class="s1">'/Users/{username}/{directory}/omnisharp-roslyn/artifacts/publish/OmniSharp.Stdio.Driver/mono/OmniSharp.exe'</span>
<span class="c">"" The roslyn server releases come with an embedded Mono, but this can be overridden to use the installed Mono by setting g:OmniSharp_server_use_mono</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_server_use_mono</span> <span class="p">=</span> <span class="m">1</span>
<span class="c">"" Timeout in seconds to wait for a response from the server</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_timeout</span> <span class="p">=</span> <span class="m">1</span>
<span class="c">"" Get Code Issues and syntax errors</span>
<span class="k">let</span> <span class="nv">g:syntastic_cs_checkers</span> <span class="p">=</span> <span class="p">[</span><span class="s1">'syntax'</span><span class="p">,</span> <span class="s1">'semantic'</span><span class="p">,</span> <span class="s1">'issues'</span><span class="p">]</span>
augroup omnisharp_commands
autocmd<span class="p">!</span>
<span class="c">"" Set autocomplete function to OmniSharp (if not using YouCompleteMe completion plugin)</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> <span class="k">setlocal</span> <span class="nb">omnifunc</span><span class="p">=</span>OmniSharp#Complete
<span class="c">"" automatic syntax check on events (TextChanged requires Vim 7.4)</span>
autocmd <span class="nb">BufEnter</span><span class="p">,</span><span class="nb">TextChanged</span><span class="p">,</span><span class="nb">InsertLeave</span> *<span class="p">.</span><span class="k">cs</span> SyntasticCheck
<span class="c">"" The following commands are contextual, based on the current cursor position.</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="nb">gd</span> <span class="p">:</span>OmniSharpGotoDefinition<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fi <span class="p">:</span>OmniSharpFindImplementations<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">ft</span> <span class="p">:</span>OmniSharpFindType<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">fs</span> <span class="p">:</span>OmniSharpFindSymbol<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">fu</span> <span class="p">:</span>OmniSharpFindUsages<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" finds members in the current buffer</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fm <span class="p">:</span>OmniSharpFindMembers<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" cursor can be anywhere on the line containing an issue</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fx <span class="p">:</span>OmniSharpFixUsings<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>tt <span class="p">:</span>OmniSharpTypeLookup<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>dc <span class="p">:</span>OmniSharpDocumentation<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" navigate up by method/property/field</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>C<span class="p">-</span>K<span class="p">></span> <span class="p">:</span>OmniSharpNavigateUp<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" navigate down by method/property/field</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>C<span class="p">-</span>J<span class="p">></span> <span class="p">:</span>OmniSharpNavigateDown<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Contextual code actions (requires CtrlP or unite.vim)</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">a</span> <span class="p">:</span>OmniSharpGetCodeActions<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Run code actions with text selected in visual mode to extract method</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> vnoremap <span class="p"><</span>space<span class="p">></span><span class="k">a</span> <span class="p">:</span><span class="k">call</span> OmniSharp#GetCodeActions<span class="p">(</span><span class="s1">'visual'</span><span class="p">)<</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" rename with dialog</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>rm <span class="p">:</span>OmniSharpRename<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Force OmniSharp to reload the solution. Useful when switching branches etc.</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">cf</span> <span class="p">:</span>OmniSharpCodeFormat<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" (Experimental - uses vim-dispatch or vimproc plugin) - Restart the omnisharp server for the current solution</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">rs</span> <span class="p">:</span>OmniSharpRestartServer<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Add syntax highlighting for types and interfaces</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">th</span> <span class="p">:</span>OmniSharpHighlightTypes<span class="p"><</span><span class="k">cr</span><span class="p">></span>
augroup END</code></pre></figure>
<p>If you have ale plugin installed, make sure to set ale_linters with OmniSharp, otherwise you’ll got many lint errors.</p>
<figure class="highlight"><pre><code class="language-vimscript" data-lang="vimscript"><span class="c">"" only enable these linters</span>
<span class="k">let</span> <span class="nv">g:ale_linters</span> <span class="p">=</span> <span class="p">{</span>
<span class="se"> \</span> <span class="s1">'javascript'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'eslint'</span><span class="p">],</span>
<span class="se"> \</span> <span class="s1">'cs'</span><span class="p">:</span> <span class="p">[</span><span class="s1">'OmniSharp'</span><span class="p">]</span>
<span class="se"> \</span><span class="p">}</span></code></pre></figure>
<h2 id="unity3d-development-in-macvim">Unity3D development in MacVim</h2>
<ul>
<li>
<p>Right click *.cs file from the Finder, select “Open With” -> “MacVim” as the default Editor in your system.</p>
</li>
<li>
<p>Set Unity3D Preferences -> External Tools -> External Script Editor to “Open by file extension”.</p>
</li>
<li>
<p>Create/Delete/Rename/Move C# script files from Unity3D Editor (Which will handle the .meta files).</p>
</li>
<li>
<p>Double click C# script in Unity3D project view to open MacVim for development.</p>
</li>
</ul>
<p>If you don’t want to set MacVim as system default editor. You can also set <em>MacVim.app</em> as Unity3D default editor. But if you double-click scripts in Unity3D project view multiple times, Unity3D will create multiple MacVim applications (You’ll see them while pressing Command + Tab).</p>
<p>That’s what we don’t want to see. We only want to have one MacVim application when press Command + Tab and can switch workspaces with Command + ~. The following steps can help.</p>
<ul>
<li>
<p>Create a bash script <em>open_macvim.sh</em> which start MacVim. Its content can simply be <code class="language-plaintext highlighter-rouge">mvim $@</code></p>
</li>
<li>
<p>Download <a href="https://gist.github.com/mathiasbynens/674099">appify</a> script.</p>
</li>
<li>
<p>Type the following commands:</p>
</li>
</ul>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="nb">chmod </span>a+x appify
<span class="nb">chmod </span>a+x open_macvim.sh
./applify open_macvim.sh <span class="s2">"open_macvim"</span></code></pre></figure>
<ul>
<li>
<p>Then you’ll get an <em>open_macvim.app</em></p>
</li>
<li>
<p>Select open_macvim.app from Unity3D as default editor.</p>
</li>
</ul>
<h2 id="reference">Reference</h2>
<p><a href="https://github.com/OmniSharp/omnisharp-roslyn">omnisharp-roslyn</a></p>
<p><a href="https://libraries.io/github/OmniSharp/omnisharp-vim">omnisharp-vim</a></p>
<p><a href="https://github.com/OmniSharp/omnisharp-vim/wiki/Unity3D-and-.NET-Framework">Unity3D and .NET Framework</a></p>
<p><a href="https://docs.unity3d.com/Manual/CommandLineArguments.html">Unity3D Command line arguments</a></p>
<p><a href="https://github.com/OmniSharp/omnisharp-vim/issues/506">Issue: Doesn’t download OmniSharp-Roslyn before trying to start on macOS</a></p>
<p><a href="https://github.com/coc-extensions/coc-omnisharp">coc-omnisharp</a></p>
<p><a href="https://stackoverflow.com/questions/49550056/unity-opens-multiple-workspaces-when-using-vscode-as-editor">Unity opens multiple workspaces when using VSCode as editor</a></p>Compile omnisharp-roslyn serverUse nvim in Unity3d on MacOS2021-01-22T08:31:00+00:002021-01-22T08:31:00+00:00/unity3d/2021/01/22/use-nvim-in-unity3d-on-macos<h2 id="compile-omnisharp-roslyn-server">Compile omnisharp-roslyn server</h2>
<p>Clone <a href="https://github.com/OmniSharp/omnisharp-roslyn">omnisharp-roslyn</a></p>
<p>Use command <code class="language-plaintext highlighter-rouge">./build.sh</code> to compile omnisharp-roslyn server (omnisharp-roslyn will be compiled with local environment mono)</p>
<h2 id="nvim-settings">nvim settings</h2>
<p>Create <code class="language-plaintext highlighter-rouge">~/.config/nvim/omnisharp.config.vim</code></p>
<p>Copy the following script to omnisharp.config.vim.</p>
<p>Don’t forget to change the value of <code class="language-plaintext highlighter-rouge">let g:OmniSharp_server_path = ''</code> to the absolute path of OmniSharp.exe which built just now.</p>
<figure class="highlight"><pre><code class="language-vimscript" data-lang="vimscript"><span class="c">" OmniSharp</span>
<span class="c">"" Compile omnisharp-roslyn locally and set the artifacts OmniSharp.exe to OmniSharp_server_path</span>
<span class="c">"" https://github.com/OmniSharp/omnisharp-roslyn</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_server_path</span> <span class="p">=</span> <span class="s1">'/Users/{username}/{directory}/omnisharp-roslyn/artifacts/publish/OmniSharp.Stdio.Driver/mono/OmniSharp.exe'</span>
<span class="c">"" The roslyn server releases come with an embedded Mono, but this can be overridden to use the installed Mono by setting g:OmniSharp_server_use_mono</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_server_use_mono</span> <span class="p">=</span> <span class="m">1</span>
<span class="c">"" Timeout in seconds to wait for a response from the server</span>
<span class="k">let</span> <span class="nv">g:OmniSharp_timeout</span> <span class="p">=</span> <span class="m">1</span>
<span class="c">"" Get Code Issues and syntax errors</span>
<span class="k">let</span> <span class="nv">g:syntastic_cs_checkers</span> <span class="p">=</span> <span class="p">[</span><span class="s1">'syntax'</span><span class="p">,</span> <span class="s1">'semantic'</span><span class="p">,</span> <span class="s1">'issues'</span><span class="p">]</span>
augroup omnisharp_commands
autocmd<span class="p">!</span>
<span class="c">"" Set autocomplete function to OmniSharp (if not using YouCompleteMe completion plugin)</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> <span class="k">setlocal</span> <span class="nb">omnifunc</span><span class="p">=</span>OmniSharp#Complete
<span class="c">"" automatic syntax check on events (TextChanged requires Vim 7.4)</span>
autocmd <span class="nb">BufEnter</span><span class="p">,</span><span class="nb">TextChanged</span><span class="p">,</span><span class="nb">InsertLeave</span> *<span class="p">.</span><span class="k">cs</span> SyntasticCheck
<span class="c">"" The following commands are contextual, based on the current cursor position.</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="nb">gd</span> <span class="p">:</span>OmniSharpGotoDefinition<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fi <span class="p">:</span>OmniSharpFindImplementations<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">ft</span> <span class="p">:</span>OmniSharpFindType<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">fs</span> <span class="p">:</span>OmniSharpFindSymbol<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">fu</span> <span class="p">:</span>OmniSharpFindUsages<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" finds members in the current buffer</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fm <span class="p">:</span>OmniSharpFindMembers<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" cursor can be anywhere on the line containing an issue</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>fx <span class="p">:</span>OmniSharpFixUsings<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>tt <span class="p">:</span>OmniSharpTypeLookup<span class="p"><</span><span class="k">cr</span><span class="p">></span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>dc <span class="p">:</span>OmniSharpDocumentation<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" navigate up by method/property/field</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>C<span class="p">-</span>K<span class="p">></span> <span class="p">:</span>OmniSharpNavigateUp<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" navigate down by method/property/field</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>C<span class="p">-</span>J<span class="p">></span> <span class="p">:</span>OmniSharpNavigateDown<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Contextual code actions (requires CtrlP or unite.vim)</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">a</span> <span class="p">:</span>OmniSharpGetCodeActions<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Run code actions with text selected in visual mode to extract method</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> vnoremap <span class="p"><</span>space<span class="p">></span><span class="k">a</span> <span class="p">:</span><span class="k">call</span> OmniSharp#GetCodeActions<span class="p">(</span><span class="s1">'visual'</span><span class="p">)<</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" rename with dialog</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span>rm <span class="p">:</span>OmniSharpRename<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Force OmniSharp to reload the solution. Useful when switching branches etc.</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">cf</span> <span class="p">:</span>OmniSharpCodeFormat<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" (Experimental - uses vim-dispatch or vimproc plugin) - Restart the omnisharp server for the current solution</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="nb">rs</span> <span class="p">:</span>OmniSharpRestartServer<span class="p"><</span><span class="k">cr</span><span class="p">></span>
<span class="c">"" Add syntax highlighting for types and interfaces</span>
autocmd <span class="nb">FileType</span> <span class="k">cs</span> nnoremap <span class="p"><</span>space<span class="p">></span><span class="k">th</span> <span class="p">:</span>OmniSharpHighlightTypes<span class="p"><</span><span class="k">cr</span><span class="p">></span>
augroup END</code></pre></figure>
<p>Add following lines to <code class="language-plaintext highlighter-rouge">~/.config/nvim/init.vim</code>.</p>
<figure class="highlight"><pre><code class="language-vimscript" data-lang="vimscript">Plug <span class="s1">'OmniSharp/omnisharp-vim'</span>
<span class="k">source</span> <span class="p">~</span><span class="sr">/.config/</span>nvim/omnisharp<span class="p">.</span>config<span class="p">.</span><span class="k">vim</span></code></pre></figure>
<p>Run <code class="language-plaintext highlighter-rouge">nvim</code>, execute <code class="language-plaintext highlighter-rouge">:PlugInstall</code> to install omnisharp-vim plugin.</p>
<h2 id="unity3d-development-in-nvim">Unity3D development in nvim</h2>
<ul>
<li>
<p>Run nvim from the directory <em>{project_dir}/Assets/Scripts/</em>.</p>
</li>
<li>
<p>Create/Delete/Rename/Move C# script files from Unity3D Editor (Which will handle the .meta files).</p>
</li>
<li>
<p>After operation of script files in Unity3D, Go to Preferences -> External Tools and Click <strong>Regenerate project files</strong> to update the sln and csproj file.</p>
</li>
<li>
<p>Use <code class="language-plaintext highlighter-rouge"><space>rs</code> in nvim to restart omnisharp server, which will reload sln and csproj files.</p>
</li>
</ul>
<h2 id="reference">Reference</h2>
<p><a href="https://github.com/OmniSharp/omnisharp-roslyn">omnisharp-roslyn</a></p>
<p><a href="https://libraries.io/github/OmniSharp/omnisharp-vim">omnisharp-vim</a></p>
<p><a href="https://github.com/OmniSharp/omnisharp-vim/wiki/Unity3D-and-.NET-Framework">Unity3D and .NET Framework</a></p>
<p><a href="https://docs.unity3d.com/Manual/CommandLineArguments.html">Unity3D Command line arguments</a></p>
<p><a href="https://github.com/OmniSharp/omnisharp-vim/issues/506">Issue: Doesn’t download OmniSharp-Roslyn before trying to start on macOS</a></p>
<p><a href="https://github.com/coc-extensions/coc-omnisharp">coc-omnisharp</a></p>Compile omnisharp-roslyn servernode express authentication2020-03-25T05:39:07+00:002020-03-25T05:39:07+00:00/web/2020/03/25/node-express-authentication<h2 id="introduction">Introduction</h2>
<p>This article is about how to do authentication with node express.</p>
<h3 id="add-packages-to-your-project">Add packages to your project</h3>
<p><em>package.json</em></p>
<figure class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="w">
</span><span class="nl">"dependencies"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="nl">"express"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^4.17.1"</span><span class="p">,</span><span class="w">
</span><span class="nl">"body-parser"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^1.19.0"</span><span class="p">,</span><span class="w">
</span><span class="nl">"passport"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^0.4.1"</span><span class="p">,</span><span class="w">
</span><span class="nl">"passport-http"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^0.3.0"</span><span class="p">,</span><span class="w">
</span><span class="nl">"passport-strategy"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^1.0.0"</span><span class="p">,</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span></code></pre></figure>
<h3 id="create-auth-module">Create Auth module</h3>
<p>Create an <em>Auth.js</em> module.</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="kd">const</span> <span class="nx">passport</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="dl">'</span><span class="s1">passport</span><span class="dl">'</span><span class="p">);</span>
<span class="kd">const</span> <span class="nx">Strategy</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="dl">'</span><span class="s1">passport-http</span><span class="dl">'</span><span class="p">).</span><span class="nx">DigestStrategy</span><span class="p">;</span>
<span class="kd">const</span> <span class="nx">STRATEGY_DIGEST</span> <span class="o">=</span> <span class="dl">'</span><span class="s1">digest</span><span class="dl">'</span><span class="p">;</span>
<span class="kd">class</span> <span class="nx">Auth</span> <span class="p">{</span>
<span class="kd">constructor</span><span class="p">(</span><span class="nx">dbManager</span><span class="p">)</span> <span class="p">{</span>
<span class="k">this</span><span class="p">.</span><span class="nx">_dbManager</span> <span class="o">=</span> <span class="nx">dbManager</span><span class="p">;</span>
<span class="k">this</span><span class="p">.</span><span class="nx">_passport</span> <span class="o">=</span> <span class="nx">passport</span><span class="p">;</span>
<span class="k">this</span><span class="p">.</span><span class="nx">init</span><span class="p">();</span>
<span class="p">}</span>
<span class="nx">init</span><span class="p">()</span> <span class="p">{</span>
<span class="k">this</span><span class="p">.</span><span class="nx">_passport</span><span class="p">.</span><span class="nx">use</span><span class="p">(</span><span class="nx">STRATEGY_DIGEST</span><span class="p">,</span> <span class="k">new</span> <span class="nx">Strategy</span><span class="p">({</span> <span class="na">qop</span><span class="p">:</span> <span class="dl">'</span><span class="s1">auth</span><span class="dl">'</span> <span class="p">},</span>
<span class="k">async</span> <span class="p">(</span><span class="nx">username</span><span class="p">,</span> <span class="nx">cb</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="kd">const</span> <span class="nx">users</span> <span class="o">=</span> <span class="k">await</span> <span class="k">this</span><span class="p">.</span><span class="nx">_dbManager</span><span class="p">.</span><span class="nx">userDao</span><span class="p">.</span><span class="nx">findUsers</span><span class="p">(</span><span class="nx">username</span><span class="p">);</span>
<span class="k">if</span> <span class="p">(</span><span class="nx">users</span><span class="p">.</span><span class="nx">length</span> <span class="o">></span> <span class="mi">0</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">cb</span><span class="p">(</span><span class="kc">null</span><span class="p">,</span> <span class="nx">users</span><span class="p">[</span><span class="mi">0</span><span class="p">].</span><span class="nx">username</span><span class="p">,</span> <span class="nx">users</span><span class="p">[</span><span class="mi">0</span><span class="p">].</span><span class="nx">password</span><span class="p">);</span>
<span class="p">}</span> <span class="k">else</span> <span class="p">{</span>
<span class="nx">cb</span><span class="p">(</span><span class="kc">null</span><span class="p">,</span> <span class="kc">false</span><span class="p">);</span>
<span class="p">}</span>
<span class="p">}));</span>
<span class="p">}</span>
<span class="kd">get</span> <span class="nx">digestAuth</span><span class="p">()</span> <span class="p">{</span>
<span class="k">return</span> <span class="k">this</span><span class="p">.</span><span class="nx">_passport</span><span class="p">.</span><span class="nx">authenticate</span><span class="p">(</span><span class="nx">STRATEGY_DIGEST</span><span class="p">,</span> <span class="p">{</span> <span class="na">session</span><span class="p">:</span> <span class="kc">false</span> <span class="p">});</span>
<span class="p">}</span>
<span class="p">}</span>
<span class="nx">module</span><span class="p">.</span><span class="nx">exports</span> <span class="o">=</span> <span class="nx">Auth</span><span class="p">;</span></code></pre></figure>
<p>If your service is deployed on cloud with envoy proxy, the DigestStrategy return 400 BadRequest, you have to customize the <em>digest.js</em> of passport-http library.</p>
<p>Put the following code to digest.js url checking part to deal with envoy proxy problem.</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="kd">const</span> <span class="nx">envoyOriginalPath</span> <span class="o">=</span> <span class="nx">req</span><span class="p">.</span><span class="nx">headers</span><span class="p">[</span><span class="dl">'</span><span class="s1">x-envoy-original-path</span><span class="dl">'</span><span class="p">];</span>
<span class="kd">const</span> <span class="nx">url</span> <span class="o">=</span> <span class="nx">envoyOriginalPath</span> <span class="o">||</span> <span class="nx">req</span><span class="p">.</span><span class="nx">originalUrl</span> <span class="o">||</span> <span class="nx">req</span><span class="p">.</span><span class="nx">url</span><span class="p">;</span>
<span class="k">if</span> <span class="p">(</span><span class="nx">url</span> <span class="o">!==</span> <span class="nx">creds</span><span class="p">.</span><span class="nx">uri</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="k">this</span><span class="p">.</span><span class="nx">fail</span><span class="p">(</span><span class="mi">400</span><span class="p">);</span>
<span class="p">}</span></code></pre></figure>
<h3 id="use-auth-strategy-on-api">Use auth strategy on API</h3>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="kd">const</span> <span class="nx">express</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="dl">'</span><span class="s1">express</span><span class="dl">'</span><span class="p">);</span>
<span class="kd">const</span> <span class="nx">bodyParser</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="dl">'</span><span class="s1">body-parser</span><span class="dl">'</span><span class="p">);</span>
<span class="kd">const</span> <span class="nx">Auth</span> <span class="o">=</span> <span class="nx">require</span><span class="p">(</span><span class="dl">'</span><span class="s1">./auth/Auth</span><span class="dl">'</span><span class="p">);</span>
<span class="kd">const</span> <span class="nx">app</span> <span class="o">=</span> <span class="nx">express</span><span class="p">();</span>
<span class="nx">app</span><span class="p">.</span><span class="nx">use</span><span class="p">(</span><span class="nx">bodyParser</span><span class="p">.</span><span class="nx">json</span><span class="p">());</span>
<span class="kd">const</span> <span class="nx">userService</span> <span class="o">=</span> <span class="k">new</span> <span class="nx">UserService</span><span class="p">();</span>
<span class="nx">app</span><span class="p">.</span><span class="kd">get</span><span class="p">(</span><span class="dl">'</span><span class="s1">/api/user/fetchUsers</span><span class="dl">'</span><span class="p">,</span> <span class="nx">auth</span><span class="p">.</span><span class="nx">digestAuth</span><span class="p">,</span> <span class="nx">userService</span><span class="p">.</span><span class="nx">fetchUsers</span><span class="p">.</span><span class="nx">bind</span><span class="p">(</span><span class="nx">userService</span><span class="p">)));</span>
<span class="c1">// start server</span>
<span class="nx">app</span><span class="p">.</span><span class="nx">listen</span><span class="p">(</span><span class="mi">8080</span><span class="p">,</span> <span class="p">()</span> <span class="o">=></span> <span class="nx">console</span><span class="p">.</span><span class="nx">log</span><span class="p">(</span><span class="s2">`service is listening on port 8080`</span><span class="p">));</span></code></pre></figure>
<h3 id="request-api-from-client-side">Request API from client side</h3>
<p>If client side is node.js, use <a href="https://github.com/request/request">request</a> library to do api request.</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="nx">request</span><span class="p">.</span><span class="nx">post</span><span class="p">({</span>
<span class="na">uri</span><span class="p">:</span> <span class="s2">`http://localhost:8080/</span><span class="p">${</span><span class="nx">api</span><span class="p">}</span><span class="s2">`</span><span class="p">,</span>
<span class="na">headers</span><span class="p">:</span> <span class="p">{</span>
<span class="dl">'</span><span class="s1">Content-Type</span><span class="dl">'</span><span class="p">:</span> <span class="dl">'</span><span class="s1">application/json</span><span class="dl">'</span><span class="p">,</span>
<span class="p">},</span>
<span class="na">body</span><span class="p">:</span> <span class="p">{</span>
<span class="c1">// params</span>
<span class="p">},</span>
<span class="na">json</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">auth</span><span class="p">:</span> <span class="p">{</span>
<span class="na">user</span><span class="p">:</span> <span class="nx">process</span><span class="p">.</span><span class="nx">env</span><span class="p">.</span><span class="nx">USER_NAME</span><span class="p">,</span>
<span class="na">pass</span><span class="p">:</span> <span class="nx">process</span><span class="p">.</span><span class="nx">env</span><span class="p">.</span><span class="nx">PASSWORD</span><span class="p">,</span>
<span class="na">sendImmediatey</span><span class="p">:</span> <span class="kc">false</span><span class="p">,</span> <span class="c1">// Digest authentication is supported, but it only works with sendImmediately set to false (sendImmediately defaults to true, which causes a basic authentication header to be sent).</span>
<span class="p">},</span>
<span class="p">},</span> <span class="p">(</span><span class="nx">error</span><span class="p">,</span> <span class="nx">response</span><span class="p">,</span> <span class="nx">body</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="c1">// deal with response or error </span>
<span class="p">});</span></code></pre></figure>
<h3 id="references">References</h3>
<p><a href="https://stackoverflow.com/questions/24284377/bad-request-on-digest-authentication-user-false">Bad request on digest authentication (User: false)</a></p>Introductionnode mysql migration2020-03-25T04:14:07+00:002020-03-25T04:14:07+00:00/web/2020/03/25/node-mysql-migration<h2 id="introduction">Introduction</h2>
<p>This article is about how to do mysql migration in node.js project.</p>
<h2 id="setup-mysql-databse">Setup mysql databse</h2>
<p>We can setup a docker mysql for example.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">docker run <span class="nt">--name</span> mysql <span class="nt">-e</span> <span class="nv">MYSQL_ROOT_PASSWORD</span><span class="o">=</span>root <span class="nt">-p</span> 3306:3306 <span class="nt">-d</span> mysql:5.7</code></pre></figure>
<p>after docker container started.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">run <span class="s1">'mysql -h 127.0.0.1 -u root -p'</span>
<span class="nb">type</span> <span class="s1">'root'</span>
CREATE DATABASE mydb CHARACTER SET utf8 COLLATE utf8_general_ci<span class="p">;</span></code></pre></figure>
<h3 id="add-packages-to-your-project-and-configure-migrate-scripts">Add packages to your project and configure migrate scripts</h3>
<p><em>package.json</em></p>
<figure class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="w">
</span><span class="nl">"scripts"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="nl">"start"</span><span class="p">:</span><span class="w"> </span><span class="s2">"node_modules/db-migrate/bin/db-migrate up && node src/main.js"</span><span class="p">,</span><span class="w">
</span><span class="nl">"migrate-up"</span><span class="p">:</span><span class="w"> </span><span class="s2">"node_modules/db-migrate/bin/db-migrate up"</span><span class="p">,</span><span class="w">
</span><span class="nl">"migrate-down"</span><span class="p">:</span><span class="w"> </span><span class="s2">"node_modules/db-migrate/bin/db-migrate down"</span><span class="p">,</span><span class="w">
</span><span class="p">},</span><span class="w">
</span><span class="nl">"dependencies"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="nl">"mysql"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^2.17.1"</span><span class="p">,</span><span class="w">
</span><span class="nl">"db-migrate"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^0.11.6"</span><span class="p">,</span><span class="w">
</span><span class="nl">"db-migrate-mysql"</span><span class="p">:</span><span class="w"> </span><span class="s2">"^2.1.0"</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span></code></pre></figure>
<p>When you run <code class="language-plaintext highlighter-rouge">npm start</code>, migration-up will check all migration files and upgrade to the latest before application start.</p>
<p>When you run <code class="language-plaintext highlighter-rouge">npm run migrate-down</code>, the current database would only downgrade the latest migration in database. If you want to migrate-down 3 migrations, you should run <code class="language-plaintext highlighter-rouge">npm run migrate-down</code> 3 times.</p>
<h3 id="create-databasejson">Create database.json</h3>
<p>Create a <em>database.json</em> file to the root of project. With database.json, db-migrate would know how to connect to database. <a href="https://db-migrate.readthedocs.io/en/latest/Getting%20Started/configuration/">db-migrate Configuration</a></p>
<p><em>database.json</em></p>
<figure class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="w">
</span><span class="nl">"dev"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="nl">"host"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nl">"ENV"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"MYSQL_HOST"</span><span class="w"> </span><span class="p">},</span><span class="w">
</span><span class="nl">"user"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nl">"ENV"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"MYSQL_USER"</span><span class="w"> </span><span class="p">},</span><span class="w">
</span><span class="nl">"password"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nl">"ENV"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"MYSQL_PASSWORD"</span><span class="w"> </span><span class="p">},</span><span class="w">
</span><span class="nl">"database"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="nl">"ENV"</span><span class="w"> </span><span class="p">:</span><span class="w"> </span><span class="s2">"MYSQL_DATABASE"</span><span class="w"> </span><span class="p">},</span><span class="w">
</span><span class="nl">"driver"</span><span class="p">:</span><span class="w"> </span><span class="s2">"mysql"</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span></code></pre></figure>
<p>Make sure you have exported MYSQL_HOST, MYSQL_USER, MYSQL_PASSWORD and MYSQL_DATABASE to environment.</p>
<p>If the environment is not specified by the -e or –env option (db-migrate up –config config/database.json -e prod), db-migrate will look for an environment named dev or development. You can change this default behavior with the database.json file:</p>
<figure class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="w">
</span><span class="nl">"defaultEnv"</span><span class="p">:</span><span class="w"> </span><span class="s2">"local"</span><span class="p">,</span><span class="w">
</span><span class="nl">"local"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w">
</span><span class="nl">"driver"</span><span class="p">:</span><span class="w"> </span><span class="s2">"sqlite3"</span><span class="p">,</span><span class="w">
</span><span class="nl">"filename"</span><span class="p">:</span><span class="w"> </span><span class="s2">":memory:"</span><span class="w">
</span><span class="p">}</span><span class="w">
</span><span class="p">}</span></code></pre></figure>
<h3 id="create-migration-files">Create migration files</h3>
<p>Create a directory <em>migrations</em> to the root of project.</p>
<p>Create a script new-migration.sh</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">node_modules/db-migrate/bin/db-migrate create <span class="nv">$1</span></code></pre></figure>
<p>Execute <code class="language-plaintext highlighter-rouge">./new-migration.sh create-table-user</code></p>
<p>There will be a new migration file <em>{timestamp}-create-table-user.js</em> generated under ./migrations/</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="dl">'</span><span class="s1">use strict</span><span class="dl">'</span><span class="p">;</span>
<span class="kd">var</span> <span class="nx">dbm</span><span class="p">;</span>
<span class="kd">var</span> <span class="nx">type</span><span class="p">;</span>
<span class="kd">var</span> <span class="nx">seed</span><span class="p">;</span>
<span class="cm">/**
* We receive the dbmigrate dependency from dbmigrate initially.
* This enables us to not have to rely on NODE_PATH.
*/</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">setup</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">options</span><span class="p">,</span> <span class="nx">seedLink</span><span class="p">)</span> <span class="p">{</span>
<span class="nx">dbm</span> <span class="o">=</span> <span class="nx">options</span><span class="p">.</span><span class="nx">dbmigrate</span><span class="p">;</span>
<span class="nx">type</span> <span class="o">=</span> <span class="nx">dbm</span><span class="p">.</span><span class="nx">dataType</span><span class="p">;</span>
<span class="nx">seed</span> <span class="o">=</span> <span class="nx">seedLink</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">up</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="kc">null</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">down</span> <span class="o">=</span> <span class="kd">function</span><span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="p">{</span>
<span class="k">return</span> <span class="kc">null</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">_meta</span> <span class="o">=</span> <span class="p">{</span>
<span class="dl">"</span><span class="s2">version</span><span class="dl">"</span><span class="p">:</span> <span class="mi">1</span>
<span class="p">};</span></code></pre></figure>
<p>Then you should implement the exports.up function and exports.down function such as</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="dl">'</span><span class="s1">use strict</span><span class="dl">'</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">dbm</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">type</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">seed</span><span class="p">;</span>
<span class="cm">/**
* We receive the dbmigrate dependency from dbmigrate initially.
* This enables us to not have to rely on NODE_PATH.
*/</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">setup</span> <span class="o">=</span> <span class="p">(</span><span class="nx">options</span><span class="p">,</span> <span class="nx">seedLink</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">dbm</span> <span class="o">=</span> <span class="nx">options</span><span class="p">.</span><span class="nx">dbmigrate</span><span class="p">;</span>
<span class="nx">type</span> <span class="o">=</span> <span class="nx">dbm</span><span class="p">.</span><span class="nx">dataType</span><span class="p">;</span>
<span class="nx">seed</span> <span class="o">=</span> <span class="nx">seedLink</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">up</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">createTable</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">,</span> <span class="p">{</span>
<span class="na">id</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">int</span><span class="dl">'</span><span class="p">,</span>
<span class="na">primaryKey</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">unsigned</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">notNull</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">autoIncrement</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">length</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="p">},</span>
<span class="na">type</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">string</span><span class="dl">'</span><span class="p">,</span>
<span class="na">notNull</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">length</span><span class="p">:</span> <span class="mi">10</span><span class="p">,</span>
<span class="p">},</span>
<span class="na">username</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">string</span><span class="dl">'</span><span class="p">,</span>
<span class="na">notNull</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">length</span><span class="p">:</span> <span class="mi">32</span><span class="p">,</span>
<span class="na">unique</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="p">},</span>
<span class="na">password</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">string</span><span class="dl">'</span><span class="p">,</span>
<span class="na">notNull</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="na">length</span><span class="p">:</span> <span class="mi">32</span><span class="p">,</span>
<span class="p">},</span>
<span class="na">createdAt</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">timestamp</span><span class="dl">'</span><span class="p">,</span>
<span class="na">defaultValue</span><span class="p">:</span> <span class="nb">String</span><span class="p">(</span><span class="dl">'</span><span class="s1">CURRENT_TIMESTAMP</span><span class="dl">'</span><span class="p">),</span>
<span class="p">},</span>
<span class="na">updatedAt</span><span class="p">:</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">timestamp</span><span class="dl">'</span><span class="p">,</span>
<span class="na">defaultValue</span><span class="p">:</span> <span class="nb">String</span><span class="p">(</span><span class="dl">'</span><span class="s1">CURRENT_TIMESTAMP</span><span class="dl">'</span><span class="p">),</span>
<span class="p">},</span>
<span class="p">}).</span><span class="nx">then</span><span class="p">(()</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">db</span><span class="p">.</span><span class="nx">runSql</span><span class="p">(</span><span class="dl">'</span><span class="s1">ALTER TABLE `user` CHANGE COLUMN `updatedAt` `updatedAt` TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP AFTER `createdAt`;</span><span class="dl">'</span><span class="p">);</span>
<span class="p">});</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">down</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">dropTable</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">_meta</span> <span class="o">=</span> <span class="p">{</span>
<span class="na">version</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">};</span></code></pre></figure>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="cm">/* eslint-disable no-underscore-dangle */</span>
<span class="cm">/* eslint-disable arrow-body-style */</span>
<span class="cm">/* eslint-disable no-new-wrappers */</span>
<span class="dl">'</span><span class="s1">use strict</span><span class="dl">'</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">dbm</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">type</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">seed</span><span class="p">;</span>
<span class="cm">/**
* We receive the dbmigrate dependency from dbmigrate initially.
* This enables us to not have to rely on NODE_PATH.
*/</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">setup</span> <span class="o">=</span> <span class="p">(</span><span class="nx">options</span><span class="p">,</span> <span class="nx">seedLink</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">dbm</span> <span class="o">=</span> <span class="nx">options</span><span class="p">.</span><span class="nx">dbmigrate</span><span class="p">;</span>
<span class="nx">type</span> <span class="o">=</span> <span class="nx">dbm</span><span class="p">.</span><span class="nx">dataType</span><span class="p">;</span>
<span class="nx">seed</span> <span class="o">=</span> <span class="nx">seedLink</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">up</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">addColumn</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">,</span> <span class="dl">'</span><span class="s1">phone</span><span class="dl">'</span><span class="p">,</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">string</span><span class="dl">'</span><span class="p">,</span>
<span class="na">length</span><span class="p">:</span> <span class="mi">20</span><span class="p">,</span>
<span class="p">});</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">down</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">removeColumn</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">,</span> <span class="dl">'</span><span class="s1">phone</span><span class="dl">'</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">_meta</span> <span class="o">=</span> <span class="p">{</span>
<span class="na">version</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">};</span></code></pre></figure>
<p>You can always do <em>db.runSql()</em> if syntax of db-migrate cannot feed your demand.</p>
<figure class="highlight"><pre><code class="language-javascript" data-lang="javascript"><span class="cm">/* eslint-disable no-underscore-dangle */</span>
<span class="cm">/* eslint-disable arrow-body-style */</span>
<span class="cm">/* eslint-disable no-new-wrappers */</span>
<span class="dl">'</span><span class="s1">use strict</span><span class="dl">'</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">dbm</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">type</span><span class="p">;</span>
<span class="kd">let</span> <span class="nx">seed</span><span class="p">;</span>
<span class="cm">/**
* We receive the dbmigrate dependency from dbmigrate initially.
* This enables us to not have to rely on NODE_PATH.
*/</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">setup</span> <span class="o">=</span> <span class="p">(</span><span class="nx">options</span><span class="p">,</span> <span class="nx">seedLink</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">dbm</span> <span class="o">=</span> <span class="nx">options</span><span class="p">.</span><span class="nx">dbmigrate</span><span class="p">;</span>
<span class="nx">type</span> <span class="o">=</span> <span class="nx">dbm</span><span class="p">.</span><span class="nx">dataType</span><span class="p">;</span>
<span class="nx">seed</span> <span class="o">=</span> <span class="nx">seedLink</span><span class="p">;</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">up</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">addColumn</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">,</span> <span class="dl">'</span><span class="s1">notes</span><span class="dl">'</span><span class="p">,</span> <span class="p">{</span>
<span class="na">type</span><span class="p">:</span> <span class="dl">'</span><span class="s1">text</span><span class="dl">'</span><span class="p">,</span>
<span class="na">notNull</span><span class="p">:</span> <span class="kc">true</span><span class="p">,</span>
<span class="p">}).</span><span class="nx">then</span><span class="p">(()</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">db</span><span class="p">.</span><span class="nx">runSql</span><span class="p">(</span><span class="dl">"</span><span class="s2">UPDATE user SET notes='';</span><span class="dl">"</span><span class="p">);</span>
<span class="p">}).</span><span class="nx">then</span><span class="p">(()</span> <span class="o">=></span> <span class="p">{</span>
<span class="nx">db</span><span class="p">.</span><span class="nx">runSql</span><span class="p">(</span><span class="dl">'</span><span class="s1">ALTER TABLE user MODIFY COLUMN notes TEXT NOT NULL;</span><span class="dl">'</span><span class="p">);</span>
<span class="p">});</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">down</span> <span class="o">=</span> <span class="p">(</span><span class="nx">db</span><span class="p">)</span> <span class="o">=></span> <span class="p">{</span>
<span class="k">return</span> <span class="nx">db</span><span class="p">.</span><span class="nx">removeColumn</span><span class="p">(</span><span class="dl">'</span><span class="s1">user</span><span class="dl">'</span><span class="p">,</span> <span class="dl">'</span><span class="s1">notes</span><span class="dl">'</span><span class="p">);</span>
<span class="p">};</span>
<span class="nx">exports</span><span class="p">.</span><span class="nx">_meta</span> <span class="o">=</span> <span class="p">{</span>
<span class="na">version</span><span class="p">:</span> <span class="mi">1</span><span class="p">,</span>
<span class="p">};</span></code></pre></figure>
<h2 id="references">References</h2>
<p><a href="https://github.com/db-migrate/node-db-migrate">db-migrate github</a></p>
<p><a href="https://db-migrate.readthedocs.io/en/latest/">db-migrate doc</a></p>
<p><a href="https://db-migrate.readthedocs.io/en/latest/Getting%20Started/configuration/">db-migrate Configuration</a></p>
<p><a href="https://db-migrate.readthedocs.io/en/latest/Getting%20Started/usage/#creating-migrations">createing migrations</a></p>Introductionmqtt broker on android2020-03-23T06:47:00+00:002020-03-23T06:47:00+00:00/iot/2020/03/23/mqtt-broker-on-android<h2 id="introduction">Introduction</h2>
<p>On android device, we often use <a href="https://github.com/eclipse/paho.mqtt.java">paho mqtt client</a> to setup mqtt client.</p>
<p>If you want android device to become a mqtt broker. you can use <a href="https://moquette-io.github.io/moquette">Moquette MQTT broker</a>.</p>
<p>This article is about how to setup mqtt broker on android device with moqutte.</p>
<h2 id="android-app-settings">Android app settings</h2>
<p>Following settings is not mandatory, it’s just the pre-settings of the sample app.</p>
<p>language: Kotlin</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">minSdkVersion</span> <span class="mi">26</span>
<span class="n">minSdkVersion</span> <span class="mi">29</span>
<span class="n">multiDexEnabled</span> <span class="kc">true</span>
<span class="n">compileOptions</span> <span class="o">{</span>
<span class="n">sourceCompatibility</span> <span class="n">JavaVersion</span><span class="o">.</span><span class="na">VERSION_1_8</span>
<span class="n">targetCompatibility</span> <span class="n">JavaVersion</span><span class="o">.</span><span class="na">VERSION_1_8</span>
<span class="o">}</span></code></pre></figure>
<figure class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><uses-permission</span> <span class="na">android:name=</span><span class="s">"android.permission.INTERNET"</span> <span class="nt">/></span></code></pre></figure>
<h2 id="steps">Steps</h2>
<h3 id="add-dependency">Add dependency</h3>
<p>Add dependency to app build.gradle.</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">dependencies</span> <span class="o">{</span>
<span class="n">implementation</span> <span class="s1">'io.moquette:moquette-broker:0.12.1'</span>
<span class="o">}</span></code></pre></figure>
<h3 id="configure-packagingoptions">Configure packagingOptions</h3>
<p>Sync gradle, then you might got errors such as:</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">More than one file was found with OS independent path <span class="s1">'META-INF/io.netty.versions.properties'</span>
More than one file was found with OS independent path <span class="s1">'META-INF/INDEX.LIST'</span></code></pre></figure>
<p>Exclude them in build.gradle of app:</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">android</span> <span class="o">{</span>
<span class="o">...</span>
<span class="n">packagingOptions</span> <span class="o">{</span>
<span class="n">exclude</span> <span class="s1">'META-INF/io.netty.versions.properties'</span>
<span class="n">exclude</span> <span class="s1">'META-INF/INDEX.LIST'</span>
<span class="o">}</span>
<span class="o">}</span></code></pre></figure>
<h3 id="start--stop-server">Start & stop server</h3>
<figure class="highlight"><pre><code class="language-kotlin" data-lang="kotlin"><span class="k">private</span> <span class="k">fun</span> <span class="nf">startMqttBroker</span><span class="p">()</span> <span class="p">{</span>
<span class="kd">val</span> <span class="py">properties</span> <span class="p">=</span> <span class="nc">Properties</span><span class="p">()</span>
<span class="n">properties</span><span class="p">.</span><span class="nf">setProperty</span><span class="p">(</span><span class="s">"host"</span><span class="p">,</span> <span class="s">"0.0.0.0"</span><span class="p">)</span>
<span class="n">properties</span><span class="p">.</span><span class="nf">setProperty</span><span class="p">(</span><span class="s">"port"</span><span class="p">,</span> <span class="s">"1883"</span><span class="p">)</span>
<span class="n">mqttBroker</span><span class="p">.</span><span class="nf">startServer</span><span class="p">(</span><span class="n">properties</span><span class="p">)</span>
<span class="p">}</span>
<span class="k">private</span> <span class="k">fun</span> <span class="nf">stopMqttBroker</span><span class="p">()</span> <span class="p">{</span>
<span class="n">mqttBroker</span><span class="p">.</span><span class="nf">stopServer</span><span class="p">()</span>
<span class="p">}</span></code></pre></figure>
<p>It’s already ok for mqttBroker. If you want to make more detailed configurations, follow the next step.</p>
<h3 id="configure">Configure</h3>
<p>Check the properties in mosquitto.conf, and set it to mqttBroker.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c">##############################################</span>
<span class="c"># Moquette configuration file.</span>
<span class="c">#</span>
<span class="c"># The synthax is equals to mosquitto.conf</span>
<span class="c">#</span>
<span class="c">##############################################</span>
port 1883
websocket_port 8080
host 0.0.0.0
<span class="c">#false to accept only client connetions with credentials</span>
<span class="c">#true to accept client connection without credentails, validating only the one that provides</span>
allow_anonymous <span class="nb">true</span>
<span class="c">#false to prohibit clients from connecting without a clientid.</span>
<span class="c">#true to allow clients to connect without a clientid. One will be generated for them.</span>
allow_zero_byte_client_id <span class="nb">false</span></code></pre></figure>
<h2 id="references">References</h2>
<p><a href="https://stackoverflow.com/questions/28623707/mqtt-broker-for-android">mqtt-broker-for-android</a></p>
<p><a href="https://www.hivemq.com/tags/mqtt-client-library">mqtt-client-library</a></p>
<p><a href="https://github.com/eclipse/paho.mqtt.java">paho mqtt client</a></p>
<p><a href="https://moquette-io.github.io/moquette">Moquette MQTT broker</a></p>
<p><a href="https://www.eclipse.org/mosquitto/man/mosquitto-conf-5.php">mosquitto-conf</a></p>
<p><a href="https://github.com/eclipse/mosquitto/blob/master/mosquitto.conf">mosquitto.conf</a></p>Introductionprotobuf on android2020-03-23T05:46:07+00:002020-03-23T05:46:07+00:00/android/2020/03/23/protobuf-on-android<h2 id="introduction">Introduction</h2>
<p>Protocol buffers are Google’s language-neutral, platform-neutral, extensible mechanism for serializing structured data – think XML, but smaller, faster, and simpler. You define how you want your data to be structured once, then you can use special generated source code to easily write and read your structured data to and from a variety of data streams and using a variety of languages.</p>
<p>This article is about how to setup protobuf on android.</p>
<h2 id="android-app-settings">Android app settings</h2>
<p>Following settings is not mandatory, it’s just the pre-settings of the sample app.</p>
<p>language: Kotlin</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">minSdkVersion</span> <span class="mi">26</span>
<span class="n">minSdkVersion</span> <span class="mi">29</span>
<span class="n">multiDexEnabled</span> <span class="kc">true</span>
<span class="n">compileOptions</span> <span class="o">{</span>
<span class="n">sourceCompatibility</span> <span class="n">JavaVersion</span><span class="o">.</span><span class="na">VERSION_1_8</span>
<span class="n">targetCompatibility</span> <span class="n">JavaVersion</span><span class="o">.</span><span class="na">VERSION_1_8</span>
<span class="o">}</span></code></pre></figure>
<h2 id="steps">Steps</h2>
<h3 id="add-plugin-for-gradle">Add plugin for gradle</h3>
<p>The latest version is 0.8.12. It requires at least Gradle 5.6 and Java 8. It is available on Maven Central. To add dependency to it in the build.gradle of root:</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">buildscript</span> <span class="o">{</span>
<span class="n">repositories</span> <span class="o">{</span>
<span class="n">mavenCentral</span><span class="o">()</span>
<span class="o">}</span>
<span class="n">dependencies</span> <span class="o">{</span>
<span class="n">classpath</span> <span class="s1">'com.google.protobuf:protobuf-gradle-plugin:0.8.12'</span>
<span class="o">}</span>
<span class="o">}</span></code></pre></figure>
<h3 id="add-plugin-to-your-project">Add plugin to your project</h3>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">apply</span> <span class="nl">plugin:</span> <span class="s1">'com.android.application'</span> <span class="c1">// or 'com.android.library'</span>
<span class="n">apply</span> <span class="nl">plugin:</span> <span class="s1">'com.google.protobuf'</span></code></pre></figure>
<h3 id="customizing-source-directories">Customizing source directories</h3>
<ul>
<li>
<p>Create directory <em>app/src/main/proto</em>.</p>
</li>
<li>
<p>Put *.proto files to <em>app/src/main/proto</em>.</p>
</li>
</ul>
<h3 id="add-dependency">Add dependency</h3>
<p>Add dependency to app build.gradle and configure protobuf tasks.</p>
<figure class="highlight"><pre><code class="language-groovy" data-lang="groovy"><span class="n">dependencies</span> <span class="o">{</span>
<span class="c1">// You need to depend on the lite runtime library, not protobuf-java</span>
<span class="n">compile</span> <span class="s1">'com.google.protobuf:protobuf-javalite:3.8.0'</span>
<span class="o">}</span>
<span class="n">protobuf</span> <span class="o">{</span>
<span class="n">protoc</span> <span class="o">{</span>
<span class="n">artifact</span> <span class="o">=</span> <span class="s1">'com.google.protobuf:protoc:3.8.0'</span>
<span class="o">}</span>
<span class="n">generateProtoTasks</span> <span class="o">{</span>
<span class="n">all</span><span class="o">().</span><span class="na">each</span> <span class="o">{</span> <span class="n">task</span> <span class="o">-></span>
<span class="n">task</span><span class="o">.</span><span class="na">builtins</span> <span class="o">{</span>
<span class="n">java</span> <span class="o">{</span>
<span class="n">option</span> <span class="s2">"lite"</span>
<span class="o">}</span>
<span class="o">}</span>
<span class="o">}</span>
<span class="o">}</span>
<span class="o">}</span></code></pre></figure>
<p>Everything’s ok, clean project and build. you can use the generated class.</p>
<h2 id="references">References</h2>
<p><a href="https://developers.google.com/protocol-buffers">protocol-buffers</a></p>
<p><a href="https://github.com/google/protobuf-gradle-plugin">protobuf-gradle-plugin</a></p>Introductionmount volume to ec22019-03-30T04:00:07+00:002019-03-30T04:00:07+00:00/ops/2019/03/30/mount-volume-to-ec2<h2 id="prepare-mount-volume">Prepare mount volume</h2>
<p>First you need to create a volume and attach it to an existing ec2 instance on AWS console.</p>
<p><img src="/assets/img/create-volume.webp" alt="" /></p>
<h2 id="mount-volume-to-ec2">Mount volume to ec2</h2>
<p>ssh to ec2, check volume with command <code class="language-plaintext highlighter-rouge">sudo fdisk -l</code></p>
<p><img src="/assets/img/fdisk.webp" alt="" /></p>
<p>This is the 100GB volume you just attached.</p>
<p>format it with command <code class="language-plaintext highlighter-rouge">sudo mkfs.ext4 /dev/nvme1n1</code></p>
<p>Then mount it to your expected path</p>
<p><code class="language-plaintext highlighter-rouge">sudo mount /dev/nvme1n1 /var/lib/docker</code></p>
<p>(e.g. <em>/var/lib/docker</em>)</p>
<h2 id="mount-volume-automaticly-after-reboot">Mount volume automaticly after reboot</h2>
<p>Above way of mount point will lost after reboot. You might need a solution to mount volume automaticly after every reboot.</p>
<p>You need to change <em>/etc/fstab</em>, so you should keep a backup <code class="language-plaintext highlighter-rouge">sudo cp /etc/fstab /etc/fstab.orig</code></p>
<p>Then check volume UUID with command <code class="language-plaintext highlighter-rouge">sudo blkid</code></p>
<p><img src="/assets/img/blkid.webp" alt="" /></p>
<p>Edit fstab <code class="language-plaintext highlighter-rouge">sudo vim /etc/fstab</code></p>
<p>Append <code class="language-plaintext highlighter-rouge">UUID=f74db2e5-575d-427c-a907-d3167a273b52 /var/lib/docker ext4 defaults,nofail 0 2</code></p>
<h2 id="check-validity">Check validity</h2>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="nb">sudo </span>umount /var/lib/docker
<span class="nb">sudo </span>mount <span class="nt">-a</span>
<span class="nb">df</span> <span class="nt">-h</span></code></pre></figure>
<p>If you can see 100GB mount point as following figure, then it’ll works on every reboot.</p>
<p><img src="/assets/img/dfh.webp" alt="" /></p>
<h2 id="references">References</h2>
<p><a href="https://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/ebs-using-volumes.html">Making an Amazon EBS Volume Available for Use on Linux</a></p>
<p><a href="https://docs.aws.amazon.com/en_us/AWSEC2/latest/UserGuide/ebs-using-volumes.html#ebs-mount-after-reboot">Automatically Mount an Attached Volume After Reboot</a></p>
<p><a href="https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/deployment_guide/s1-filesystem-ext4-create">CREATING AN EXT4 FILE SYSTEM</a></p>
<p><a href="https://blog.csdn.net/mxy_0223/article/details/70146153">在Amazon EC2中挂载EBS作为永久存储</a></p>Prepare mount volumesetup docker registry with jenkins2019-03-17T20:38:53+00:002019-03-17T20:38:53+00:00/ops/2019/03/17/setup-docker-registry-with-jenkins<h2 id="setup-docker-registry">Setup docker registry</h2>
<p>You can setup docker registry simply by a single command</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="nb">sudo </span>docker run <span class="nt">-d</span> <span class="nt">-p</span> 5000:5000 <span class="nt">-v</span> <span class="sb">`</span><span class="nb">pwd</span><span class="sb">`</span>/data:/var/lib/registry <span class="nt">--restart</span><span class="o">=</span>always <span class="nt">--name</span> registry registry:latest</code></pre></figure>
<p>If you want to delete images pushed to registry, you’d better mapping the <em>config.yml</em> locally, because you can enable delete in <em>config.yml</em>.</p>
<p>After registry setup, you can curl to check the catalog and taglist of images.</p>
<p>Here’s a python3 script which can not only setup docker registry as a service but also access registry information / push image as a client.</p>
<p><a href="https://github.com/jie-meng/toolscripts/blob/master/docker/registry/docker-registry.py">docker registry script</a></p>
<h3 id="http--https">http & https</h3>
<p>If you setup docker registry on a server with https url, that would be easy to push image to registry. Otherwise you would got error message <code class="language-plaintext highlighter-rouge">server gave HTTP response to HTTPS client</code> when push image.</p>
<p>Then you should add registry ip:port to insecure-registries of docker client.</p>
<p>For linux, go to <em>/etc/docker/</em> and create file <em>daemon.json</em>, add <code class="language-plaintext highlighter-rouge">{ "insecure-registries": ["ip:port"] }</code>. Then <code class="language-plaintext highlighter-rouge">service docker restart</code>.</p>
<p>For Mac, open docker client preference, edit daemon as following figure.</p>
<h2 id="setup-jenkins">Setup jenkins</h2>
<p>If you want to use docker in jenkins, DO NOT use <a href="https://github.com/jenkinsci/docker">jenkins-docker</a>. That means you either need to install docker in jenkins docker or mapping host docker to jenkins. But neither of them is good choice.</p>
<p>You should install jenkins without docker.</p>
<p>On ubuntu, several commands would be OK. <a href="https://github.com/jie-meng/toolscripts/blob/master/jenkins/install-jenkins-ubuntu.sh">install-jenkins-ubuntu</a></p>
<p>Then you can operate it simply like <a href="https://github.com/jie-meng/toolscripts/blob/master/jenkins/jenkins-service-op.md">jenkins-service-op</a>.</p>
<h3 id="give-jenkins-permission-of-host-docker">Give jenkins permission of host docker</h3>
<p>Give permission of docker to jenkins user on host, or you’ll error like <code class="language-plaintext highlighter-rouge">/var/run/docker.sock: connect: permission denied.</code></p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="nb">sudo </span>usermod <span class="nt">-aG</span> docker jenkins
<span class="nb">sudo </span>service jenkins restart</code></pre></figure>
<h2 id="ci-pipeline-with-docker-registry">CI Pipeline with docker registry</h2>
<p>We have already setup docker-registry and jenkins, then we can setup a pipeline to clone project from github, build it on jenkins, push built image to registry.</p>
<p>Then we need to ssh to target machine, pull and run the image we just pushed.</p>
<h3 id="install-ssh-plugins">Install ssh plugins</h3>
<p>2 plugins need to be installed on jenkins first: <a href="https://stackoverflow.com/questions/18227009/jenkins-can-the-execute-shell-execute-ssh-commands">SSH plugin & Publish Over SSH Plugin</a>.</p>
<h3 id="give-jenkins-permission-to-ssh-to-target-machine">Give jenkins permission to ssh to target machine</h3>
<ul>
<li>
<p>On Jenkins host, <code class="language-plaintext highlighter-rouge">sudo su -s /bin/bash jenkins</code>, <code class="language-plaintext highlighter-rouge">ssh-keygen</code>, <code class="language-plaintext highlighter-rouge">cat /var/lib/jenkins/.ssh/id_rsa.pub</code>, we got jenkins ssh public key.</p>
</li>
<li>
<p>Add public key of jenkins to target machine’s <code class="language-plaintext highlighter-rouge">~/.ssh/authorized_keys</code>.</p>
</li>
<li>
<p>Login jenkins with <code class="language-plaintext highlighter-rouge">sudo su -s /bin/bash jenkins</code> on jenkins host, ssh to target machine mannualy first time, type ‘yes’ then jenkins can ssh to target machine freely on pipeline.</p>
</li>
</ul>
<h3 id="jenkins-pipeline-execute-shell-sample">Jenkins pipeline execute shell sample</h3>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">docker build <span class="nt">-t</span> my-server <span class="nb">.</span>
docker tag my-server:latest docker-registry.domain.net/my-server-qa:<span class="k">${</span><span class="nv">BUILD_NUMBER</span><span class="k">}</span>
docker push docker-registry.domain.net/my-server-qa:<span class="k">${</span><span class="nv">BUILD_NUMBER</span><span class="k">}</span>
ssh ubuntu@ec2-00-11-22-33.us-west-1.compute.amazonaws.com <span class="s2">"~/deploy-docker.sh qa </span><span class="k">${</span><span class="nv">BUILD_NUMBER</span><span class="k">}</span><span class="s2">"</span></code></pre></figure>
<h3 id="deploy-script-on-target-machine">deploy script on target machine</h3>
<p>Target machine should install docker first, otherwise it cannot pull image from docker registry.</p>
<p>This is a sample deploy script of target machine which matches above execute shell sample.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c"># param1: env (prod|stg|qa|local)</span>
<span class="c"># param2: build_number (1|2|3...)</span>
<span class="nb">sudo </span>docker pull docker-registry.domain.net/my-server-<span class="nv">$1</span>:<span class="nv">$2</span>
<span class="nb">sudo </span>docker container stop <span class="si">$(</span><span class="nb">sudo </span>docker ps <span class="nt">-f</span> <span class="nv">name</span><span class="o">=</span>my-server <span class="nt">-q</span><span class="si">)</span>
<span class="nb">set</span> <span class="nt">-e</span>
<span class="nb">sudo </span>docker run <span class="nt">--name</span> my-server <span class="nt">--rm</span> <span class="nt">-d</span> <span class="nt">-p</span> 8080:8080 docker-registry.domain.net/my-server-<span class="nv">$1</span>:<span class="nv">$2</span> <span class="nv">$1</span>
<span class="nb">sudo </span>docker system prune <span class="nt">-a</span> <span class="c"># save disk volume for server</span></code></pre></figure>
<h2 id="references">References</h2>
<p><a href="https://tonybai.com/2016/02/26/deploy-a-private-docker-registry/">部署私有Docker Registry</a></p>
<p><a href="https://github.com/jie-meng/toolscripts/blob/master/docker/registry/docker-registry.py">docker registry script</a></p>
<p><a href="https://www.jianshu.com/p/fef890c4d1c2">Docker学习之Docker Registry</a></p>
<p><a href="https://github.com/docker/distribution/issues/1874">Private registry push fail: server gave HTTP response to HTTPS client</a></p>
<p><a href="https://github.com/jenkinsci/docker">jenkins-docker</a></p>
<p><a href="http://jpetazzo.github.io/2015/09/03/do-not-use-docker-in-docker-for-ci/">Using Docker-in-Docker for your CI or testing environment? Think twice</a></p>
<p><a href="https://www.cnblogs.com/morang/p/9536622.html">Jenkins在shell脚本运行docker权限报错解决</a></p>
<p><a href="https://stackoverflow.com/questions/18227009/jenkins-can-the-execute-shell-execute-ssh-commands">Jenkins - can the “Execute Shell” execute SSH commands</a></p>
<p><a href="https://stackoverflow.com/questions/15174194/jenkins-host-key-verification-failed">Jenkins Host key verification failed</a></p>
<p><a href="http://note.qidong.name/2017/06/26/docker-clean/">清理Docker的container,image与volume</a></p>Setup docker registryAOSP partially build and test on emulator2019-02-25T03:29:57+00:002019-02-25T03:29:57+00:00/android/2019/02/25/aosp-partially-build-and-test-on-emulator<ul>
<li>
<p>After build AOSP, start emulator with command <code class="language-plaintext highlighter-rouge">emulator -writable-system</code></p>
</li>
<li>
<p>Run <code class="language-plaintext highlighter-rouge">adb remount</code> every time after emulator restart.</p>
</li>
<li>
<p>Now you can use <code class="language-plaintext highlighter-rouge">mm</code> command to partially build modules of AOSP and use <code class="language-plaintext highlighter-rouge">adb push</code> to replace jars or apps of framework.</p>
</li>
<li>
<p>You should always use <code class="language-plaintext highlighter-rouge">mm</code> to compile</p>
</li>
</ul>
<p>For example, if you modified <em>/frameworks/base/core/res/res/values/dimens.xml</em>, then you can find its nearest make file <em>Android.bp</em> under <em>/frameworks/base/core/res/</em>, then <code class="language-plaintext highlighter-rouge">mm</code> here.</p>
<p>After build, you’ll see <code class="language-plaintext highlighter-rouge">[100% 8/8] Install: out/target/product/generic_x86/system/framework/framework-res.apk</code>, then push <em>framework-res.apk</em> to emulator path <em>/system/framework/</em>. Restart emulator.</p>
<p>You can do this simply by <code class="language-plaintext highlighter-rouge">adb sync</code>. Put the following code into a shell script <em>adbsync</em>. Just run it after <code class="language-plaintext highlighter-rouge">mm</code>. Emulator will sync the build outputs and restart.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">adb remount
adb <span class="nb">sync
</span>adb shell stop
adb shell start</code></pre></figure>
<h2 id="references">References</h2>
<p><a href="https://android.stackexchange.com/questions/110927/how-to-mount-system-rewritable-or-read-only-rw-ro">How to mount /system rewritable or read-only? (RW/RO)</a></p>
<p><a href="http://blog.udinic.com/2014/05/24/aosp-part-1-get-the-code-using-the-manifest-and-repo/">AOSP Part 1: Get the code using the Manifest and Repo tool</a></p>
<p><a href="http://blog.udinic.com/2014/06/04/aosp-part-2-build-variants">AOSP Part 2: Build variants</a></p>
<p><a href="http://blog.udinic.com/2014/07/24/aosp-part-3-developing-efficiently">AOSP Part 3: Developing Efficiently</a></p>After build AOSP, start emulator with command emulator -writable-systemAOSP pre install App2019-02-20T21:16:44+00:002019-02-20T21:16:44+00:00/android/2019/02/20/aosp-pre-install-app<h2 id="target">Target</h2>
<p>Pre install android app into android ROM</p>
<h2 id="prepare-aosp-build-environment">Prepare AOSP build environment</h2>
<p>Follow steps of <a href="https://source.android.com/setup/build/requirements">AOSP setup</a>.</p>
<p>Our sample environment:</p>
<p><strong>branch</strong>: master (repo init -u https://android.googlesource.com/platform/manifest)</p>
<p><strong>target</strong>: aosp_x86-eng</p>
<p><strong>device</strong>: emulator</p>
<p>Init env every time open a new terminal:</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="nb">source</span> ./build/envsetup.sh
lunch aosp_x86-eng</code></pre></figure>
<h2 id="build-an-apk-for-pre-install">Build an APK for pre-install</h2>
<p>Use Android Studio to build an apk. I built a <em>TestMan.apk</em>.</p>
<h2 id="place-apk-to-aosp-pre-install-apps">Place apk to aosp pre-install apps</h2>
<ul>
<li>
<p>Make a dir <em>TestMan</em> in <em>AOSP_WORKSPACE/packages/apps</em>.</p>
</li>
<li>
<p>Put <em>TestMan.apk</em> here.</p>
</li>
<li>
<p>Make an <em>Android.mk</em> in <em>AOSP_WORKSPACE/packages/apps/TestMan</em>.</p>
</li>
</ul>
<p>Android.mk:</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">LOCAL_PATH :<span class="o">=</span> <span class="si">$(</span>call my-dir<span class="si">)</span>
include <span class="si">$(</span>CLEAR_VARS<span class="si">)</span>
LOCAL_PRIVILEGED_MODULE :<span class="o">=</span> <span class="nb">true
</span>LOCAL_MODULE :<span class="o">=</span> TestMan
LOCAL_SRC_FILES :<span class="o">=</span> <span class="si">$(</span>LOCAL_MODULE<span class="si">)</span>.apk
LOCAL_MODULE_CLASS :<span class="o">=</span> APPS
LOCAL_MODULE_TAGS :<span class="o">=</span> optional
LOCAL_MODULE_SUFFIX :<span class="o">=</span> <span class="si">$(</span>COMMON_ANDROID_PACKAGE_SUFFIX<span class="si">)</span>
LOCAL_CERTIFICATE :<span class="o">=</span> PRESIGNED
LOCAL_MODULE_PATH :<span class="o">=</span> <span class="si">$(</span>TARGET_OUT_APPS<span class="si">)</span>
include <span class="si">$(</span>BUILD_PREBUILT<span class="si">)</span></code></pre></figure>
<p>Each line should not have trailing spaces, or you would get error <code class="language-plaintext highlighter-rouge">Invalid characters in module stem \(LOCAL_INSTALLED_MODULE_STEM\)</code> when you build.</p>
<h2 id="modify-makefile">Modify makefile</h2>
<p>Open <em>AOSP_WORKSPACE/build/target/product/aosp_x86.mk</em>.</p>
<p>aosp_x86.mk:</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c"># GSI for system/product</span>
<span class="si">$(</span>call inherit-product, <span class="si">$(</span>SRC_TARGET_DIR<span class="si">)</span>/product/gsi_common.mk<span class="si">)</span>
<span class="c"># Emulator for vendor</span>
<span class="si">$(</span>call inherit-product-if-exists, device/generic/goldfish/x86-vendor.mk<span class="si">)</span>
<span class="si">$(</span>call inherit-product, <span class="si">$(</span>SRC_TARGET_DIR<span class="si">)</span>/product/emulator_vendor.mk<span class="si">)</span>
<span class="si">$(</span>call inherit-product, <span class="si">$(</span>SRC_TARGET_DIR<span class="si">)</span>/board/generic_x86/device.mk<span class="si">)</span>
<span class="c"># Enable mainline checking for excat this product name</span>
ifeq <span class="o">(</span>aosp_x86,<span class="si">$(</span>TARGET_PRODUCT<span class="si">)</span><span class="o">)</span>
PRODUCT_ENFORCE_ARTIFACT_PATH_REQUIREMENTS :<span class="o">=</span> relaxed
endif
PRODUCT_NAME :<span class="o">=</span> aosp_x86
PRODUCT_DEVICE :<span class="o">=</span> generic_x86
PRODUCT_BRAND :<span class="o">=</span> Android
PRODUCT_MODEL :<span class="o">=</span> AOSP on x86</code></pre></figure>
<p>We can see there is <em>$(SRC_TARGET_DIR)/product/gsi_common.mk</em> which every {TARGET} .mk files call.</p>
<p>Then we need to add <em>TestMan</em> to Default AOSP packages in makefile.</p>
<p>Open <em>AOSP_WORKSPACE/build/target/product/gsi_common.mk</em></p>
<p>We can see</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c"># Default AOSP packages</span>
PRODUCT_PACKAGES +<span class="o">=</span> <span class="se">\</span>
PhotoTable <span class="se">\</span>
WAPPushManager <span class="se">\</span></code></pre></figure>
<p>Add TestMan here</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c"># Default AOSP packages</span>
PRODUCT_PACKAGES +<span class="o">=</span> <span class="se">\</span>
PhotoTable <span class="se">\</span>
WAPPushManager <span class="se">\</span>
TestMan <span class="se">\</span></code></pre></figure>
<p>Add <em>TestMan</em> to whitelist in <em>gsi_common.mk</em>.</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell"><span class="c"># The mainline checking whitelist, should be clean up</span>
PRODUCT_ARTIFACT_PATH_REQUIREMENT_WHITELIST :<span class="o">=</span> <span class="se">\</span>
system/app/messaging/messaging.apk <span class="se">\</span>
system/app/PhotoTable/PhotoTable.apk <span class="se">\</span>
system/app/WAPPushManager/WAPPushManager.apk <span class="se">\</span>
system/app/TestMan/TestMan.apk <span class="se">\</span>
system/bin/healthd <span class="se">\</span>
system/etc/init/healthd.rc <span class="se">\</span>
system/etc/seccomp_policy/crash_dump.%.policy <span class="se">\</span>
system/etc/seccomp_policy/mediacodec.policy <span class="se">\</span>
system/etc/vintf/manifest/manifest_healthd.xml <span class="se">\</span>
system/lib/libframesequence.so <span class="se">\</span>
system/lib/libgiftranscode.so <span class="se">\</span>
system/lib64/libframesequence.so <span class="se">\</span>
system/lib64/libgiftranscode.so <span class="se">\</span>
system/priv-app/Dialer/Dialer.apk <span class="se">\</span></code></pre></figure>
<h2 id="build">Build</h2>
<ul>
<li>
<p>Go to <em>AOSP_WORKSPACE</em>, run <code class="language-plaintext highlighter-rouge">make -j8</code> or <code class="language-plaintext highlighter-rouge">make -j16</code> … (depend on your CPU cores)</p>
</li>
<li>
<p>If you got <code class="language-plaintext highlighter-rouge">#### build completed successfully (02:17 (mm:ss)) ####</code>, run <code class="language-plaintext highlighter-rouge">emulator</code>, you can see <em>TestMan</em> is already an in pre-install app which cannot be uninstall.</p>
</li>
</ul>
<p><img src="/assets/img/android-pre-install-app.webp" alt="" /></p>
<h2 id="how-to-replace-default-launcher-with-customized-launcher">How to replace default launcher with customized launcher</h2>
<h3 id="find-default-launcher-of-aosp">Find default launcher of AOSP</h3>
<p>Launcher is also a pre-install app, so we can find it under <em>AOSP_WORKSPACE/packages/apps/</em>.</p>
<p>We can see there are Luncher2 and Luncher3, why there are two launchers?</p>
<p>We can check their Android.mk to see what’s going on. Launcher2 does not have Android.mk, but Launcher3 has.</p>
<p>We can see there is <code class="language-plaintext highlighter-rouge">LOCAL_OVERRIDES_PACKAGES := Home Launcher2 Launcher3 Launcher3QuickStep</code>. Check what is <strong>LOCAL_OVERRIDES_PACKAGES</strong>.</p>
<p>From <a href="https://stackoverflow.com/questions/10579827/how-do-i-add-apks-in-an-aosp-build">How do I add APKs in an AOSP build?</a> we know that Launcher3 module hide <code class="language-plaintext highlighter-rouge">Home Launcher2 Launcher3 Launcher3QuickStep</code>.</p>
<p>So why does Launcher3 hide itself? Check <em>Android.mk</em> again we can also see <code class="language-plaintext highlighter-rouge">LOCAL_PACKAGE_NAME := Launcher3QuickStepGo</code>.</p>
<p>From <a href="http://android.mk/">Android.mk</a> we see:</p>
<p><strong>LOCAL_PACKAGE_NAME</strong> is the name of an app. For example, Dialer, Contacts, etc.</p>
<p><strong>LOCAL_MODULE</strong> is the name of what’s supposed to be generated from your Android.mk. For exmample, for libkjs, the LOCAL_MODULE is “libkjs” (the build system adds the appropriate suffix – .so .dylib .dll). For app modules, use LOCAL_PACKAGE_NAME instead of LOCAL_MODULE.</p>
<p>So we should always use <strong>LOCAL_PACKAGE_NAME</strong> when pre-install module is an App and build with source code. But if we use apk instead of source code, we should always use <strong>LOCAL_MODULE</strong>, otherwise you’ll get error when build.</p>
<p>Finally, we can see Launcher3’s real name is <strong>Launcher3QuickStepGo</strong>, and there are other <strong>LOCAL_PACKAGE_NAME</strong> in Android.mk of Launcher3: Launcher3, Launcher3Go, Launcher3QuickStep. We should use <code class="language-plaintext highlighter-rouge">LOCAL_OVERRIDES_PACKAGES := Home Launcher2 Launcher3 Launcher3QuickStep Launcher3QuickStepGo</code> in our own launcher app’s Android.mk to hide them all.</p>
<h3 id="build-new-launcher-apk-to-replace-default-launcher">Build new launcher apk to replace default launcher</h3>
<p>Then we need to build a launcher apk and place it to <em>AOSP_WORKSPACE/packages/apps/</em> with the same way of pre-install common apk metioned above. Modify gsi_common.mk, add it to product_packages and whitelist.</p>
<p>Android.mk of NewLauncher:</p>
<figure class="highlight"><pre><code class="language-shell" data-lang="shell">LOCAL_PATH :<span class="o">=</span> <span class="si">$(</span>call my-dir<span class="si">)</span>
include <span class="si">$(</span>CLEAR_VARS<span class="si">)</span>
LOCAL_PRIVILEGED_MODULE :<span class="o">=</span> <span class="nb">true
</span>LOCAL_MODULE :<span class="o">=</span> NewLauncher
LOCAL_SRC_FILES :<span class="o">=</span> <span class="si">$(</span>LOCAL_MODULE<span class="si">)</span>.apk
LOCAL_MODULE_CLASS :<span class="o">=</span> APPS
LOCAL_MODULE_TAGS :<span class="o">=</span> optional
LOCAL_MODULE_SUFFIX :<span class="o">=</span> <span class="si">$(</span>COMMON_ANDROID_PACKAGE_SUFFIX<span class="si">)</span>
LOCAL_CERTIFICATE :<span class="o">=</span> PRESIGNED
LOCAL_MODULE_PATH :<span class="o">=</span> <span class="si">$(</span>TARGET_OUT_APPS<span class="si">)</span>
LOCAL_OVERRIDES_PACKAGES :<span class="o">=</span> Home Launcher2 Launcher3 Launcher3Go Launcher3QuickStep Launcher3QuickStepGo
include <span class="si">$(</span>BUILD_PREBUILT<span class="si">)</span></code></pre></figure>
<h3 id="build-1">Build</h3>
<ul>
<li>
<p>Run <code class="language-plaintext highlighter-rouge">make installclean</code> to remove default launcher apk which generated under <em>AOSP_WORKSPACE/out</em> directory.</p>
</li>
<li>
<p>Run <code class="language-plaintext highlighter-rouge">make -j8</code></p>
</li>
<li>
<p>Run <code class="language-plaintext highlighter-rouge">emulator</code> to see new launcher after boot.</p>
</li>
</ul>
<p>(If you do not hide default launcher with <strong>LOCAL_OVERRIDES_PACKAGES</strong>, you will need to select a launcher after boot.)</p>
<h2 id="clean">Clean</h2>
<p>If you remove pre install app, you need to run <code class="language-plaintext highlighter-rouge">make installclean</code> before <code class="language-plaintext highlighter-rouge">make</code>.</p>
<p>If you want to clear app data, you need to run <code class="language-plaintext highlighter-rouge">make dataclean</code> before <code class="language-plaintext highlighter-rouge">make</code>.</p>
<h2 id="references">References</h2>
<p><a href="https://source.android.com/setup/build/requirements">AOSP setup</a></p>
<p><a href="https://stackoverflow.com/questions/10579827/how-do-i-add-apks-in-an-aosp-build">How do I add APKs in an AOSP build?</a></p>
<p><a href="https://stackoverflow.com/questions/22911156/how-do-i-set-the-default-launcher-in-an-aosp-build">How do I set the default launcher in an AOSP build?</a></p>
<p><a href="http://android.mk/">Android.mk</a></p>Target