Commit 6ed70fde authored by LUCAS VINCENT's avatar LUCAS VINCENT
Browse files

[WIP] Updates doc and tries to use pelican to serve static html documentaion.

parent 46d6ba15
Pipeline #17093 failed with stage
in 12 seconds
__pycache__
doc/output
docker/registry/certs
image: alpine:latest
image: 3.7.2-alpine3.8
###
# Documentation
###
pages:
script:
- apk add --update --no-cache markdown
- markdown doc/index.md > doc/index.html
- pip install -r requirements.txt
- cd doc && pelican -s publishconf.py
- mkdir public
- pwd
- ls -hal ..
- ls -hal public
- ls -hal doc
- mv doc/index.html public/.
- mv doc/output/* public/.
artifacts:
paths:
- public
PY?=python3
PELICAN?=pelican
PELICANOPTS=
BASEDIR=$(CURDIR)
INPUTDIR=$(BASEDIR)/content
OUTPUTDIR=$(BASEDIR)/output
CONFFILE=$(BASEDIR)/pelicanconf.py
PUBLISHCONF=$(BASEDIR)/publishconf.py
DEBUG ?= 0
ifeq ($(DEBUG), 1)
PELICANOPTS += -D
endif
RELATIVE ?= 0
ifeq ($(RELATIVE), 1)
PELICANOPTS += --relative-urls
endif
help:
@echo 'Makefile for a pelican Web site '
@echo ' '
@echo 'Usage: '
@echo ' make html (re)generate the web site '
@echo ' make clean remove the generated files '
@echo ' make regenerate regenerate files upon modification '
@echo ' make publish generate using production settings '
@echo ' make serve [PORT=8000] serve site at http://localhost:8000'
@echo ' make serve-global [SERVER=0.0.0.0] serve (as root) to $(SERVER):80 '
@echo ' make devserver [PORT=8000] serve and regenerate together '
@echo ' make ssh_upload upload the web site via SSH '
@echo ' make rsync_upload upload the web site via rsync+ssh '
@echo ' '
@echo 'Set the DEBUG variable to 1 to enable debugging, e.g. make DEBUG=1 html '
@echo 'Set the RELATIVE variable to 1 to enable relative urls '
@echo ' '
html:
$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
clean:
[ ! -d $(OUTPUTDIR) ] || rm -rf $(OUTPUTDIR)
regenerate:
$(PELICAN) -r $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
serve:
ifdef PORT
$(PELICAN) -l $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS) -p $(PORT)
else
$(PELICAN) -l $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
endif
serve-global:
ifdef SERVER
$(PELICAN) -l $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS) -p $(PORT) -b $(SERVER)
else
$(PELICAN) -l $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS) -p $(PORT) -b 0.0.0.0
endif
devserver:
ifdef PORT
$(PELICAN) -lr $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS) -p $(PORT)
else
$(PELICAN) -lr $(INPUTDIR) -o $(OUTPUTDIR) -s $(CONFFILE) $(PELICANOPTS)
endif
publish:
$(PELICAN) $(INPUTDIR) -o $(OUTPUTDIR) -s $(PUBLISHCONF) $(PELICANOPTS)
.PHONY: html help clean regenerate serve serve-global devserver stopserver publish
\ No newline at end of file
# How to use docker / singularity on the HPC cluster
Title: How to use docker / singularity on the HPC cluster
Date: 2019-01-24
category: hpc
Authors: David Brusson, Vincent Lucas
## Requirements
Send us your ssh public-key (cf. section "How to generate a ssh key").
## Create a singularity image
1. Create a directory and move to it.
2. Create a image definition file "<project_name>.def" (e.g. https://git.unistra.fr/HPC/singularity/blob/master/examples/simple/simple.def)
3. Generate the singularity image with the following script: hpc_login_gen.sh <project_name>
4. When successful, the image is generated as <project_name>.sif
* Create a directory and move to it.
* Create a image definition file `<project_name>.def` (e.g. [exemple.def](https://git.unistra.fr/HPC/singularity/blob/master/examples/simple/simple.def))
* Generate the singularity image with the following script:
```bash
module load singularity/singularity-3.0.3
hpc_login_gen.sh <project_name>
```
* When successful, the image is generated as `<project_name>.sif`
### Test a singularity image
```
module load singularity/singularity-2.6
### Test a singularity image (run on your PC or hpc-login)
```bash
module load singularity/singularity-3.0.3
singularity run <project_name>.sif
```
### Submit a singularity image (run on cluster nodes)
This is a simple sbatch script example:
```
```bash
#!/bin/bash
#SBATCH -N 8 # Request 8 cores
module load singularity/singularity-2.6
module load singularity/singularity-3.0.3
singularity run <project_name>.sif
```
Submit the sbatch script:
```
sbatch my_script
......@@ -32,58 +41,78 @@ sbatch my_script
## Singularity with intel18
### Create image
1. Create a image definition with the following example: https://git.unistra.fr/HPC/singularity/blob/master/examples/from_intel18/jacobi.def
2. Generate the singularity image with the following script: hpc_login_gen.sh <project_name>
3. When successful, run the image with the specific bind:
```
* Create a image definition with the following example:
[jacobi.def](https://git.unistra.fr/HPC/singularity/blob/master/examples/from_intel18/jacobi.def)
* Generate the singularity image with the following script:
```bash
hpc_login_gen.sh <project_name>
```
* When successful, run the image with the specific bind:
```bash
singularity run --bind /opt/intel/compilers_and_libraries_2018.1.163:/opt/intel/compilers_and_libraries_2018.1.163 <project_name>.sif
```
## Singularity with GPU
### Create image
1. Create a image definition with the following example: https://git.unistra.fr/HPC/singularity/blob/master/examples/gpu/gpu.def
2. Generate the singularity image with the following script: hpc_login_gen.sh <project_name>
3. When successful, run the image with the specific option:
```
* Create a image definition with the following example:
[gpu.def](https://git.unistra.fr/HPC/singularity/blob/master/examples/gpu/gpu.def)
* Generate the singularity image with the following script:
```bash
hpc_login_gen.sh <project_name>
```
* When successful, run the image with the specific option:
```bash
singularity run --nv <project_name>.sif
```
## Singularity with MPI
### Create image
1. Create a image definition with the following example: https://git.unistra.fr/HPC/singularity/blob/master/images/mpi/bandwidth.def
2. Generate the singularity image with the following script: hpc_login_gen.sh <project_name>
3. When successful, run the image with the specific option:
```
* Create a image definition with the following example:
[bandwidth.def](https://git.unistra.fr/HPC/singularity/blob/master/images/mpi/bandwidth.def)
* Generate the singularity image with the following script:
```bash
hpc_login_gen.sh <project_name>
```
* When successful, run the image with the specific option:
```bash
singularity run --bind /usr/local/openmpi/openmpi-3.1.3-i18:/usr/local/openmpi/openmpi-3.1.3-i18 --bind /opt/intel/compilers_and_libraries_2018.1.163:/opt/intel/compilers_and_libraries_2018.1.163 <project_name>.sif
```
## Transform a docker image to singularity (without MPI)
### From the docker hub
From a docker image available on docker hub, create your <project_name>.def file
with:
```
From a docker image available on docker hub, create your `<project_name>.def`
file with:
```bash
Bootstrap: docker
From: <my_image>:<my_tag>
```
### From a docker local image
1. Get an archive from your docker image
```
* Get an archive from your docker image
```bash
docker image save -o my_image.tar image_name:version
```
2. Write the corresponding singularity .def file
From a docker local image, create your <project_name>.def file
with:
* Write the corresponding singularity .def file
From a docker local image, create your `<project_name>.def` file
with:
```
Bootstrap: docker
From: hpc-singularity.di.unistra.fr:5000/<my_image>
```
3. Generate the singularity image with the following script: hpc_login_docker.sh <my_image.tar> <my_image.def>
4. When successful, the image is generated as <my_image>.sif
* Generate the singularity image with the following script:
```bash
hpc_login_docker.sh <my_image.tar> <my_image.def>
```
* When successful, the image is generated as `<my_image>.sif`
## How to generate a ssh key
```bash
ssh-keygen -t rsa -b 4096
```
## Singularity help pages
https://www.sylabs.io/guides/2.6/user-guide/
[Singularity official documentation](https://www.sylabs.io/guides/3.0/user-guide/)
#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals
AUTHOR = 'Vincent Lucas'
SITENAME = 'HPC singularity'
SITEURL = ''
PATH = 'content'
TIMEZONE = 'Europe/Paris'
DEFAULT_LANG = 'en'
# Feed generation is usually not desired when developing
FEED_ALL_ATOM = None
CATEGORY_FEED_ATOM = None
TRANSLATION_FEED_ATOM = None
AUTHOR_FEED_ATOM = None
AUTHOR_FEED_RSS = None
# Blogroll
LINKS = (
('HPC', 'https://hpc.unistra.fr/'),
('Support', 'https://support.unistra.fr/'),
('Université de Strasbourg', 'https://unistra.fr/')
)
# Social widget
#SOCIAL = (('You can add links in your config file', '#'),
# ('Another social link', '#'),)
DEFAULT_PAGINATION = 10
# Uncomment following line if you want document-relative URLs when developing
#RELATIVE_URLS = True
#!/usr/bin/env python
# -*- coding: utf-8 -*- #
from __future__ import unicode_literals
# This file is only used if you use `make publish` or
# explicitly specify it as your config file.
import os
import sys
sys.path.append(os.curdir)
from pelicanconf import *
# If your site is available via HTTPS, make sure SITEURL begins with https://
SITEURL = 'https://hpc.pages.unistra.fr/singularity/'
RELATIVE_URLS = False
#FEED_ALL_ATOM = 'feeds/all.atom.xml'
#CATEGORY_FEED_ATOM = 'feeds/{slug}.atom.xml'
DELETE_OUTPUT_DIRECTORY = True
# Following items are often useful when publishing
#DISQUS_SITENAME = ""
#GOOGLE_ANALYTICS = ""
# -*- coding: utf-8 -*-
import os
import shutil
import sys
import datetime
from invoke import task
from invoke.util import cd
from pelican.server import ComplexHTTPRequestHandler, RootedHTTPServer
CONFIG = {
# Local path configuration (can be absolute or relative to tasks.py)
'deploy_path': 'output',
# Port for `serve`
'port': 8000,
}
@task
def clean(c):
"""Remove generated files"""
if os.path.isdir(CONFIG['deploy_path']):
shutil.rmtree(CONFIG['deploy_path'])
os.makedirs(CONFIG['deploy_path'])
@task
def build(c):
"""Build local version of site"""
c.run('pelican -s pelicanconf.py')
@task
def rebuild(c):
"""`build` with the delete switch"""
c.run('pelican -d -s pelicanconf.py')
@task
def regenerate(c):
"""Automatically regenerate site upon file modification"""
c.run('pelican -r -s pelicanconf.py')
@task
def serve(c):
"""Serve site at http://localhost:8000/"""
class AddressReuseTCPServer(RootedHTTPServer):
allow_reuse_address = True
server = AddressReuseTCPServer(
CONFIG['deploy_path'],
('', CONFIG['port']),
ComplexHTTPRequestHandler)
sys.stderr.write('Serving on port {port} ...\n'.format(**CONFIG))
server.serve_forever()
@task
def reserve(c):
"""`build`, then `serve`"""
build(c)
serve(c)
@task
def preview(c):
"""Build production version of site"""
c.run('pelican -s publishconf.py')
@task
def publish(c):
"""Publish to production via rsync"""
c.run('pelican -s publishconf.py')
c.run(
'rsync --delete --exclude ".DS_Store" -pthrvz -c '
'{} {production}:{dest_path}'.format(
CONFIG['deploy_path'].rstrip('/') + '/',
**CONFIG))
# Documentation
The documentation is generated using Pelican module.
## Installation
In a python virtualenv, run the following command to install "pelican" and
"Markdown" modules:
'''
pip install -r requirements
'''
## Generates basic documentation files
'''
cd doc
pelican-quickstart -p .
'''
## Adds content
Add all your mardown files to the "doc/content" directory.
The files must follow the pelican metadata format.
## Generates html files
In "doc" folder, run '''make html'''.
## Test with a local server
In "doc" folder, run '''make serve'''.
The server is available at the following address: '''http://localhost:8000:'''.
## References
* http://docs.getpelican.com/en/3.1.1/getting_started.html
pelican
Markdown
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment