Support dev install on windows (#2416)

update development document.

docs/en_US/Tutorial/SetupNniDeveloperEnvironment.md

-**Set up NNI developer environment**
+# Setup NNI development environment
 
-===
+NNI development environment supports Ubuntu 1604 (or above), and Windows 10 with Python3 64bit.
 
-## Best practice for debug NNI source code
+## Installation
 
-For debugging NNI source code, your development environment should be under Ubuntu 16.04 (or above) system with python 3 and pip 3 installed, then follow the below steps.
+The installation steps are similar with installing from source code. But the installation links to code directory, so that code changes can be applied to installation as easy as possible.
 
-### 1. Clone the source code
+### 1. Clone source code
 
-Run the command
-
-```
+```bash
 git clone https://github.com/Microsoft/nni.git
 ```
 
-to clone the source code
+Note, if you want to contribute code back, it needs to fork your own NNI repo, and clone from there.
 
-### 2. Prepare the debug environment and install dependencies
+### 2. Install from source code
 
-Change directory to the source code folder, then run the command
+#### Ubuntu
 
+```bash
+make dev-easy-install
 ```
-make install-dependencies
-```
-
-to install the dependent tools for the environment
-
-### 3. Build source code
 
-Run the command
+#### Windows
 
+```bat
+powershell -ExecutionPolicy Bypass -file install.ps1 -Development
 ```
-make build
-```
-
-to build the source code
 
-### 4. Install NNI to development environment
-
-Run the command
-
-```
-make dev-install
-```
-
-to install the distribution content to development environment, and create cli scripts
-
-### 5. Check if the environment is ready
+### 3. Check if the environment is ready
 
 Now, you can try to start an experiment to check if your environment is ready.
 For example, run the command
 
-```
-nnictl create --config ~/nni/examples/trials/mnist-tfv1/config.yml
+```bash
+nnictl create --config examples/trials/mnist-tfv1/config.yml
 ```
 
 And open WebUI to check if everything is OK
 
-### 6. Redeploy
-
-After the code changes, it may need to redeploy. It depends on what kind of code changed. 
+### 4. Reload changes
 
 #### Python
 
-It doesn't need to redeploy, but the nnictl may need to be restarted.
+Nothing to do, the code is already linked to package folders.
 
 #### TypeScript
 
-* If `src/nni_manager` is changed, run `yarn watch` continually under this folder. It will rebuild code instantly. The nnictl may need to be restarted to reload NNI manager.
+* If `src/nni_manager` is changed, run `yarn watch` under this folder. It will watch and build code continually. The `nnictl` need to be restarted to reload NNI manager.
 * If `src/webui` or `src/nasui` are changed, run `yarn start` under the corresponding folder. The web UI will refresh automatically if code is changed.
 
+### 5. Submit Pull Request
+
+All changes are merged to master branch from your forked repo. The description of Pull Request must be meaningful, and useful.
+
+We will review the changes as soon as possible. Once it passes review, we will merge it to master branch.
 
----
-At last, wish you have a wonderful day.
-For more contribution guidelines on making PR's or issues to NNI source code, you can refer to our [Contributing](Contributing.md) document.
+For more contribution guidelines and coding styles, you can refer to the [contributing document](Contributing.md).

install.ps1

+param ([Switch] $Development)
 [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
 
+
 $install_node = $true
 $install_yarn = $true
 
-if([Environment]::Is64BitOperatingSystem){
+if ([Environment]::Is64BitOperatingSystem) {
     $OS_VERSION = 'win64'
 }
-else{
+else {
     $OS_VERSION = 'win32'
 }
 # nodejs
@@ -15,58 +17,58 @@ $yarnUrl = "https://yarnpkg.com/latest.tar.gz"
 $unzipNodeDir = "node-v*"
 $unzipYarnDir = "yarn-v*"
 
-$NNI_DEPENDENCY_FOLDER = [System.IO.Path]::GetTempPath()+$env:USERNAME
+$NNI_DEPENDENCY_FOLDER = [System.IO.Path]::GetTempPath() + $env:USERNAME
 
 $WHICH_PYTHON = where.exe python
-if($WHICH_PYTHON -eq $null){
+if ($WHICH_PYTHON -eq $null) {
     throw "Can not find python"
 }
-else{
+else {
     $pyVersion = & python -V 2>&1
-    $pyVersion =  ([string]$pyVersion).substring(7,3)
-    if([double]$pyVersion -lt 3.5){
+    $pyVersion = ([string]$pyVersion).substring(7, 3)
+    if ([double]$pyVersion -lt 3.5) {
         throw "python version should >= 3.5"
     }
 }
 
 $WHICH_PIP = where.exe pip
-if($WHICH_PIP -eq $null){
+if ($WHICH_PIP -eq $null) {
     throw "Can not find pip"
 }
 
 $env:PYTHONIOENCODING = "UTF-8"
-if($env:VIRTUAL_ENV){
+if ($env:VIRTUAL_ENV) {
     $NNI_PYTHON3 = $env:VIRTUAL_ENV + "\Scripts"
     $NNI_PKG_FOLDER = $env:VIRTUAL_ENV + "\nni"
     $NNI_PYTHON_SCRIPTS = $NNI_PYTHON3
 }
-else{
+else {
     $NNI_PYTHON3 = $(python -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]))')
     $NNI_PKG_FOLDER = $NNI_PYTHON3 + "\nni"
     $NNI_PYTHON_SCRIPTS = $NNI_PYTHON3 + "\Scripts"
 }
 
-$PIP_INSTALL = """$NNI_PYTHON3\python"" -m pip install ."
+$PIP_INSTALL = """$NNI_PYTHON3\python"" -m pip install "
 
-if(!(Test-Path $NNI_DEPENDENCY_FOLDER)){
+if (!(Test-Path $NNI_DEPENDENCY_FOLDER)) {
     New-Item $NNI_DEPENDENCY_FOLDER -ItemType Directory
 }
-$NNI_NODE_ZIP = $NNI_DEPENDENCY_FOLDER+"\nni-node.zip"
-$NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-node"
-$NNI_YARN_TARBALL = $NNI_DEPENDENCY_FOLDER+"\nni-yarn.tar.gz"
-$NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-yarn"
-$NNI_YARN = $NNI_YARN_FOLDER +"\bin\yarn"
+$NNI_NODE_ZIP = $NNI_DEPENDENCY_FOLDER + "\nni-node.zip"
+$NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER + "\nni-node"
+$NNI_YARN_TARBALL = $NNI_DEPENDENCY_FOLDER + "\nni-yarn.tar.gz"
+$NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER + "\nni-yarn"
+$NNI_YARN = $NNI_YARN_FOLDER + "\bin\yarn"
 
 ## Version number
 $NNI_VERSION_VALUE = $(git describe --tags)
 $NNI_VERSION_TEMPLATE = "999.0.0-developing"
 
-if(!(Test-Path $NNI_NODE_ZIP)){
+if (!(Test-Path $NNI_NODE_ZIP)) {
     Write-Host "Downloading Node..."
     (New-Object Net.WebClient).DownloadFile($nodeUrl, $NNI_NODE_ZIP)
 }
 
-if(!(Test-Path $NNI_YARN_TARBALL)){
+if (!(Test-Path $NNI_YARN_TARBALL)) {
     Write-Host "Downloading Yarn..."
     (New-Object Net.WebClient).DownloadFile($yarnUrl, $NNI_YARN_TARBALL)
 }
@@ -81,20 +83,23 @@ $SCRIPT =  "import tarfile",
 [System.IO.File]::WriteAllLines($SCRIPT_PATH, $SCRIPT)
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem
-function Unzip{
+function Unzip {
     param([string]$zipfile, [string]$outpath)
     [System.IO.Compression.ZipFile]::ExtractToDirectory($zipfile, $outpath)
 }
 if ($install_node) {
     ### nodejs install
-    if(!(Test-Path $NNI_NODE_FOLDER)){
+    if (!(Test-Path $NNI_NODE_FOLDER)) {
         Unzip $NNI_NODE_ZIP $NNI_DEPENDENCY_FOLDER
         $unzipNodeDir = Get-ChildItem "$NNI_DEPENDENCY_FOLDER\$unzipNodeDir"
         Rename-Item $unzipNodeDir "nni-node"
     }
     Copy-Item "$NNI_NODE_FOLDER\node.exe" $NNI_PYTHON_SCRIPTS -Recurse -Force
+}
+
+if ($install_yarn) {
     ### yarn install
-    if(!(Test-Path $NNI_YARN_FOLDER)){
+    if (!(Test-Path $NNI_YARN_FOLDER)) {
         cmd /C """$NNI_PYTHON3\python""" $SCRIPT_PATH
         $unzipYarnDir = Get-ChildItem "$NNI_DEPENDENCY_FOLDER\$unzipYarnDir"
         Rename-Item $unzipYarnDir "nni-yarn"
@@ -104,10 +109,40 @@ if ($install_node) {
 ## install-python-modules:
 ### Installing Python SDK
 (Get-Content setup.py).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content setup.py
-cmd /c $PIP_INSTALL
+
+if ($Development) {
+    $PYTHON_BUILD = "build"
+    if (Test-Path $PYTHON_BUILD) { 
+        # To compat with file and links.
+        cmd /c rmdir /s /q $PYTHON_BUILD
+    }
+    New-Item $PYTHON_BUILD -ItemType Directory
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni" -Target "src\sdk\pynni\nni"
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nnicli" -Target "src\sdk\pycli\nnicli"
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_annotation" -Target "tools\nni_annotation"
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_cmd" -Target "tools\nni_cmd"
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_trial_tool" -Target "tools\nni_trial_tool"
+    New-Item -ItemType Junction -Path "$($PYTHON_BUILD)\nni_gpu_tool" -Target "tools\nni_gpu_tool"
+
+    Copy-Item setup.py $PYTHON_BUILD
+    Copy-Item README.md $PYTHON_BUILD
+
+    Push-Location build
+    #update folders in setup file
+    (Get-Content setup.py).replace("src/sdk/pynni/", "") | Set-Content setup.py
+    (Get-Content setup.py).replace("src/sdk/pycli/", "") | Set-Content setup.py
+    (Get-Content setup.py).replace("src/sdk/pynni", ".") | Set-Content setup.py
+    (Get-Content setup.py).replace("tools/", "") | Set-Content setup.py
+    # install current folder.
+    cmd /c $PIP_INSTALL -e .
+    Pop-Location
+}
+else {
+    cmd /c $PIP_INSTALL .
+}
 
 # Building NNI Manager
-$env:PATH=$NNI_PYTHON_SCRIPTS+';'+$env:PATH
+$env:PATH = $NNI_PYTHON_SCRIPTS + ';' + $env:PATH
 cd src\nni_manager
 cmd /c $NNI_YARN
 cmd /c $NNI_YARN build
@@ -124,17 +159,31 @@ cmd /c $NNI_YARN build
 cd ..\..
 
 ## install-node-modules
-if(!(Test-Path $NNI_PKG_FOLDER)){
-    New-Item $NNI_PKG_FOLDER -ItemType Directory
+if (Test-Path $NNI_PKG_FOLDER) {
+    # it needs to remove the whole folder for following copy.
+    cmd /c rmdir /s /q $NNI_PKG_FOLDER
+}
+
+$NNI_PKG_FOLDER_STATIC = $NNI_PKG_FOLDER + "\static"
+$NASUI_PKG_FOLDER = $NNI_PKG_FOLDER + "\nasui"
+
+if ($Development) {
+    New-Item -ItemType Junction -Path $($NNI_PKG_FOLDER) -Target "src\nni_manager\dist"
+    New-Item -ItemType Junction -Path "$($NNI_PKG_FOLDER)\node_modules" -Target "src\nni_manager\node_modules"
+    New-Item -ItemType Junction -Path $($NNI_PKG_FOLDER_STATIC) -Target "src\webui\build"
+    New-Item -ItemType Junction -Path $($NASUI_PKG_FOLDER) -Target "src\nasui\build"
 }
-Remove-Item $NNI_PKG_FOLDER -Recurse -Force
-Copy-Item "src\nni_manager\dist" $NNI_PKG_FOLDER -Recurse
+else {
+    Copy-Item "src\nni_manager\dist" $NNI_PKG_FOLDER -Recurse
+    Copy-Item "src\webui\build" $NNI_PKG_FOLDER_STATIC -Recurse
+    Copy-Item "src\nasui\build" $NASUI_PKG_FOLDER -Recurse
+}
+
 Copy-Item "src\nni_manager\package.json" $NNI_PKG_FOLDER
 $PKG_JSON = $NNI_PKG_FOLDER + "\package.json"
 (Get-Content $PKG_JSON).replace($NNI_VERSION_TEMPLATE, $NNI_VERSION_VALUE) | Set-Content $PKG_JSON
-cmd /c $NNI_YARN --prod --cwd $NNI_PKG_FOLDER
-$NNI_PKG_FOLDER_STATIC = $NNI_PKG_FOLDER + "\static"
-$NASUI_PKG_FOLDER = $NNI_PKG_FOLDER + "\nasui"
-Copy-Item "src\webui\build" $NNI_PKG_FOLDER_STATIC -Recurse
-Copy-Item "src\nasui\build" $NASUI_PKG_FOLDER -Recurse
 Copy-Item "src\nasui\server.js" $NASUI_PKG_FOLDER -Recurse
+
+if (!$Development) {
+    cmd /c $NNI_YARN --prod --cwd $NNI_PKG_FOLDER
+}

uninstall.ps1

@@ -4,12 +4,12 @@ $env:PYTHONIOENCODING = "UTF-8"
 if($env:VIRTUAL_ENV){
     $NNI_PYTHON3 = $env:VIRTUAL_ENV + "\Scripts"
     $NNI_PKG_FOLDER = $env:VIRTUAL_ENV + "\nni"
-    Remove-Item "$NNI_PYTHON3\node.exe" -Force
+    cmd /c del "$NNI_PYTHON3\node.exe"
 }
 else{
     $NNI_PYTHON3 = $(python -c 'import site; from pathlib import Path; print(Path(site.getsitepackages()[0]))')
     $NNI_PKG_FOLDER = $NNI_PYTHON3 + "\nni"
-    Remove-Item "$NNI_PYTHON3\Scripts\node.exe" -Force
+    cmd /c del "$NNI_PYTHON3\Scripts\node.exe"
 }
 
 $PIP_UNINSTALL = """$NNI_PYTHON3\python"" -m pip uninstall -y "
@@ -17,13 +17,16 @@ $NNI_NODE_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-node"
 $NNI_YARN_FOLDER = $NNI_DEPENDENCY_FOLDER+"\nni-yarn"
 
 # uninstall
-Remove-Item $NNI_PKG_FOLDER -Recurse -Force
-cmd /C $PIP_UNINSTALL "nni"
+cmd /c rmdir /s /q $NNI_PKG_FOLDER
+cmd /c $PIP_UNINSTALL "nni"
 
-# clean
-Remove-Item "src/nni_manager/dist" -Recurse -Force
-Remove-Item "src/nni_manager/node_modules" -Recurse -Force
-Remove-Item "src/webui/build" -Recurse -Force
-Remove-Item "src/webui/node_modules" -Recurse -Force
-Remove-Item $NNI_YARN_FOLDER -Recurse -Force
-Remove-Item $NNI_NODE_FOLDER -Recurse -Force
+# clean up
+cmd /c rmdir /s /q "build"
+cmd /c rmdir /s /q "src\nni_manager\dist"
+cmd /c rmdir /s /q  "src\nni_manager\node_modules"
+cmd /c rmdir /s /q  "src\webui\build"
+cmd /c rmdir /s /q  "src\webui\node_modules"
+cmd /c rmdir /s /q  "src\nasui\build"
+cmd /c rmdir /s /q  "src\nasui\node_modules"
+cmd /c rmdir /s /q  $NNI_YARN_FOLDER
+cmd /c rmdir /s /q  $NNI_NODE_FOLDER