Installation of *deepG*
===
###### tags: `Howto`
## on Luna
```bash
export LC_ALL="en_US.UTF-8"
export LD_LIBRARY_PATH=LD_LIBRARY_PATH:/usr/local/cuda-10.0/lib64/
```
- Use `nvidia-smi` to check availability of GPUs on the server
### Installation of dependencies
```r
install.packages("hdf5r", configure.args="--with-hdf5=/usr/bin/h5cc")
install.packages("devtools")
install.packages("tensorflow")
library(tensorflow)
install_tensorflow(version="2.0.0-gpu", method = "conda", conda_python_version = "3.7")
```
You can test if `tensorflow` is installed correct by typing `tensorflow::tf_config()`. The expected output is:
```
TensorFlow v2.0.0 (/usr/local/lib/python2.7/dist-packages/tensorflow)
Python v3.7 (/usr/bin/python)
```
### Installation of *deepG*
```
devtools::install_github("hiddengenome/deepG")
library(deepG)
```
### Installation of additional python packages
If you need to install other python packages to same environment:
```bash
conda activate r-reticulate
pip install package_name
```
## on DGX-1
login
```bash
ssh username@grid.bifo.helmholtz-hzi.de
ssh pmuench@bioinf026
screen -r
```
install R
```bash
export TAR="/bin/tar"
conda env list
conda activate r-reticulate
conda install -c r r
which R
```
install packages
```r
install.packages("devtools")
devtools::install_github("rstudio/tensorflow")
install.packages("hdf5r", configure.args="--with-hdf5=/usr/bin/h5cc")
devtools::install_github("hiddengenome/deepG")
#install_tensorflow(version="2.0.0-gpu", method = "conda", conda_python_version = "3.7")
tensorflow::tf$`__version__`
```
```R
library(deepG)
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-single", layer.size = 512, tensorboard.log = "/vol/projects/pmuench/tensorboard")
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, layer.size = 512, gpu.num = 7, merge.on.cpu = FALSE, use.multiple.gpus = TRUE, use.cudnn = TRUE, tensorboard.log = "/vol/projects/pmuench/tensorboard", run.name ="dgx-multi")
```
## on other grid servers
This is what I needed to run to install it, e.g. you might need to change the paths
```bash
unset PYTHONPATH
export PATH=/home/pmuench/anaconda3/envs/r-retriculate/bin:/home/pmuench/anaconda3/bin:/home/pmuench/anaconda3/condabin:/usr/lib64/openmpi/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin
export LD_LIBARY_PATH=/home/pmuench/anaconda3/envs/r-retriculate/lib/
```
```r
remove.packages("deepG", lib = .libPaths())
install.packages("hdf5r", configure.args="--with-hdf5=/usr/bin/h5cc")
devtools::install_github('hiddengenome/deepG@statesWithAmbNucleotides')
library(deepG)
```
## server independent options
### disable eager mode
```
library(tensorflow)
tf$compat$v1$disable_eager_execution()
```
## multi-GPU trainings
### test with dummy generator
```R
devtools::install_github("hiddengenome/deepG@dummyGenerator")
# single
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-single-dummy", layer.size = 512, dummyGen = TRUE)
# multiple
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-mult-dummy", layer.size = 512, dummyGen = TRUE, gpu.num = 7, merge.on.cpu = FALSE, use.multiple.gpus = TRUE)
# 3 GPUS
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-mult3-dummy", layer.size = 512, dummyGen = TRUE, gpu.num = 3, merge.on.cpu = FALSE, use.multiple.gpus = TRUE)
```
### non-eager mode
sometimes e.g. for multi-GPU learning you need to disable eager mode
```r
library(deepG)
library(tensorflow)
tf$compat$v1$disable_eager_execution()
# 352ms/step with dummy on 2 GPUS
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-mult-dummy-non-eager", layer.size = 512, dummyGen = TRUE, gpu.num = 2, merge.on.cpu = FALSE, use.multiple.gpus = TRUE, steps.per.epoch = 500)
# 333ms/step without dummy on 2 GPUs
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-mult-dummy-non-eager", layer.size = 512, dummyGen = FALSE, gpu.num = 2, merge.on.cpu = FALSE, use.multiple.gpus = TRUE, steps.per.epoch = 500)
# test 1 CPU scope
trainNetwork(path = "/vol/projects/pmuench/ncbi_genomes/train/", path.val = "/vol/projects/pmuench/ncbi_genomes/test/", maxlen = 100, use.cudnn = TRUE, run.name ="dgx-mult-2", batch.size = 4096, dummyGen = TRUE, gpu.num = 8, merge.on.cpu = FALSE, use.multiple.gpus = TRUE, steps.per.epoch = 500, layer.size = 512, layers.lstm = 4,max.queue.size = 10000, use.codon.cnn =FALSE)
```
Sign in with Wallet
Connect another wallet