Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found
Select Git revision

Target

Select target project
  • sccs/docs.it4i.cz
  • soj0018/docs.it4i.cz
  • lszustak/docs.it4i.cz
  • jarosjir/docs.it4i.cz
  • strakpe/docs.it4i.cz
  • beranekj/docs.it4i.cz
  • tab0039/docs.it4i.cz
  • davidciz/docs.it4i.cz
  • gui0013/docs.it4i.cz
  • mrazek/docs.it4i.cz
  • lriha/docs.it4i.cz
  • it4i-vhapla/docs.it4i.cz
  • hol0598/docs.it4i.cz
  • sccs/docs-it-4-i-cz-fumadocs
  • siw019/docs-it-4-i-cz-fumadocs
15 results
Select Git revision
Show changes
Showing
with 1410 additions and 0 deletions
!
! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
! University Research and Technology
! Corporation. All rights reserved.
! Copyright (c) 2006-2015 Cisco Systems, Inc. All rights reserved.
! $COPYRIGHT$
!
! Simple ring test program using the Fortran mpi module bindings.
!
program ring
use mpi
implicit none
integer :: rank, size, tag, next, from, ierr, i, message
! Start up MPI
call MPI_INIT(ierr)
call MPI_COMM_RANK(MPI_COMM_WORLD, rank, ierr)
call MPI_COMM_SIZE(MPI_COMM_WORLD, size, ierr)
! Calculate the rank of the next process in the ring. Use the modulus
! operator so that the last process "wraps around" to rank zero.
tag = 201
next = mod((rank + 1), size)
from = mod((rank + size - 1), size)
! If we are the "master" process (i.e., MPI_COMM_WORLD rank 0), put
! the number of times to go around the ring in the message.
if (rank .eq. 0) then
message = 10
write(*, '("Process 0 sending ", i2, " to ", i2, " tag ", i3, " (", i2, " processes in ring)")') message, next, tag, size
call MPI_SEND(message, 1, MPI_INTEGER, next, tag, MPI_COMM_WORLD, ierr)
write(*, '("Process 0 sent to ", i2)') next
endif
! Pass the message around the ring. The exit mechanism works as
! follows: the message (a positive integer) is passed around the ring.
! Each time it passes rank 0, it is decremented. When each processes
! receives a message containing a 0 value, it passes the message on to
! the next process and then quits. By passing the 0 message first,
! every process gets the 0 message and can quit normally.
i = 1
10 call MPI_Recv(message, i, MPI_INTEGER, from, tag, MPI_COMM_WORLD, &
MPI_STATUS_IGNORE, ierr)
if (rank .eq. 0) then
message = message - 1
write(*, '("Process 0 decremented value: ", i2)') message
endif
call MPI_SEND(message, 1, MPI_INTEGER, next, tag, MPI_COMM_WORLD, ierr)
if (message .eq. 0) then
write(*, '("Process ", i2, " exiting")') rank
goto 20
endif
goto 10
! The last process does one extra send to process 0, which needs to be
! received before the program can exit
20 if (rank .eq. 0) then
call MPI_RECV(message, 1, MPI_INTEGER, from, tag, MPI_COMM_WORLD, &
MPI_STATUS_IGNORE, ierr)
endif
! All done
call MPI_FINALIZE(ierr)
end program
! -*- f90 -*-
!
! Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
! University Research and Technology
! Corporation. All rights reserved.
! Copyright (c) 2006-2015 Cisco Systems, Inc. All rights reserved.
! Copyright (c) 2009-2012 Los Alamos National Security, LLC.
! All rights reserved.
! $COPYRIGHT$
!
! Simple ring test program using the Fortran mpi_f08 module bindings.
!
program ring
use mpi_f08
implicit none
integer :: rank, size, tag, next, from, i, message
! Start up MPI
call MPI_INIT()
call MPI_COMM_RANK(MPI_COMM_WORLD, rank)
call MPI_COMM_SIZE(MPI_COMM_WORLD, size)
! Calculate the rank of the next process in the ring. Use the modulus
! operator so that the last process "wraps around" to rank zero.
tag = 201
next = mod((rank + 1), size)
from = mod((rank + size - 1), size)
! If we are the "master" process (i.e., MPI_COMM_WORLD rank 0), put
! the number of times to go around the ring in the message.
if (rank .eq. 0) then
message = 10
write(*, '("Process 0 sending ", i2, " to ", i2, " tag ", i3, " (", i2, " processes in ring)")') message, next, tag, size
call MPI_SEND(message, 1, MPI_INTEGER, next, tag, MPI_COMM_WORLD)
write(*, '("Process 0 sent to ", i2)') next
endif
! Pass the message around the ring. The exit mechanism works as
! follows: the message (a positive integer) is passed around the ring.
! Each time it passes rank 0, it is decremented. When each processes
! receives a message containing a 0 value, it passes the message on to
! the next process and then quits. By passing the 0 message first,
! every process gets the 0 message and can quit normally.
i = 1
10 call MPI_Recv(message, i, MPI_INTEGER, from, tag, MPI_COMM_WORLD, &
MPI_STATUS_IGNORE)
if (rank .eq. 0) then
message = message - 1
write(*, '("Process 0 decremented value: ", i2)') message
endif
call MPI_SEND(message, 1, MPI_INTEGER, next, tag, MPI_COMM_WORLD)
if (message .eq. 0) then
write(*, '("Process ", i2, " exiting")') rank
goto 20
endif
goto 10
! The last process does one extra send to process 0, which needs to be
! received before the program can exit
20 if (rank .eq. 0) then
call MPI_RECV(message, 1, MPI_INTEGER, from, tag, MPI_COMM_WORLD, &
MPI_STATUS_IGNORE)
endif
! All done
call MPI_FINALIZE()
end program
/*
* Copyright (c) 2018 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
*
* Simple example usage of SPCs through MPI_T.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "mpi.h"
/* Sends 'num_messages' messages of 'message_size' bytes from rank 0 to rank 1.
* All messages are send synchronously and with the same tag in MPI_COMM_WORLD.
*/
void message_exchange(int num_messages, int message_size)
{
int i, rank;
/* Use calloc to initialize data to 0's */
char *data = (char*)calloc(message_size, sizeof(char));
MPI_Status status;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if(rank == 0) {
for(i = 0; i < num_messages; i++)
MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
} else if(rank == 1) {
for(i = 0; i < num_messages; i++)
MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
}
free(data);
}
int main(int argc, char **argv)
{
int num_messages, message_size;
if(argc < 3) {
printf("Usage: mpirun -np 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true ./spc_example [num_messages] [message_size]\n");
return -1;
} else {
num_messages = atoi(argv[1]);
message_size = atoi(argv[2]);
}
int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index;
MPI_Datatype datatype;
MPI_T_enum enumtype;
MPI_Comm comm;
char name[256], description[256];
/* Counter names to be read by ranks 0 and 1 */
char *counter_names[] = {"runtime_spc_OMPI_BYTES_SENT_USER",
"runtime_spc_OMPI_BYTES_RECEIVED_USER" };
MPI_Init(NULL, NULL);
MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if(size != 2) {
fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
/* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
index = -1;
MPI_T_pvar_get_num(&num);
for(i = 0; i < num; i++) {
name_len = desc_len = 256;
PMPI_T_pvar_get_info(i, name, &name_len, &verbosity,
&var_class, &datatype, &enumtype, description, &desc_len, &bind,
&readonly, &continuous, &atomic);
if(strcmp(name, counter_names[rank]) == 0) {
index = i;
printf("[%d] %s -> %s\n", rank, name, description);
}
}
/* Make sure we found the counters */
if(index == -1) {
fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
MPI_Abort(MPI_COMM_WORLD, -1);
}
int ret;
long long value;
MPI_T_pvar_session session;
MPI_T_pvar_handle handle;
/* Create the MPI_T sessions/handles for the counters and start the counters */
ret = MPI_T_pvar_session_create(&session);
ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
ret = MPI_T_pvar_start(session, handle);
message_exchange(num_messages, message_size);
ret = MPI_T_pvar_read(session, handle, &value);
/* Print the counter values in order by rank */
for(i = 0; i < 2; i++) {
if(i == rank) {
printf("[%d] Value Read: %lld\n", rank, value);
fflush(stdout);
}
MPI_Barrier(MPI_COMM_WORLD);
}
/* Stop the MPI_T session, free the handle, and then free the session */
ret = MPI_T_pvar_stop(session, handle);
ret = MPI_T_pvar_handle_free(session, &handle);
ret = MPI_T_pvar_session_free(&session);
MPI_T_finalize();
MPI_Finalize();
return 0;
}
#!/usr/bin/env python
import numpy as np
import matplotlib.pyplot as plt
import torch
from torch.autograd import Function
from torchvision import datasets, transforms
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
import cudaq
from cudaq import spin
# GPU utilities
for tar in cudaq.get_targets():
print(f'{tar.description} {tar.name} {tar.platform} {tar.simulator} {tar.num_qpus}')
cudaq.set_target("default") # Set CUDAQ to run on GPU's
torch.cuda.is_available(
) # If this is True then the NVIDIA drivers are correctly installed
torch.cuda.device_count() # Counts the number of GPU's available
torch.cuda.current_device()
torch.cuda.get_device_name(0)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
# Training set
sample_count = 140
X_train = datasets.FashionMNIST(
root="./data",
train=True,
download=True,
transform=transforms.Compose([transforms.ToTensor()]),
)
# Leaving only labels 0 and 1
idx = np.append(
np.where(X_train.targets == 0)[0][:sample_count],
np.where(X_train.targets == 1)[0][:sample_count],
)
X_train.data = X_train.data[idx]
X_train.targets = X_train.targets[idx]
train_loader = torch.utils.data.DataLoader(X_train, batch_size=1, shuffle=True)
# Test set
sample_count = 70
X_test = datasets.FashionMNIST(
root="./data",
train=False,
download=True,
transform=transforms.Compose([transforms.ToTensor()]),
)
idx = np.append(
np.where(X_test.targets == 0)[0][:sample_count],
np.where(X_test.targets == 1)[0][:sample_count],
)
X_test.data = X_test.data[idx]
X_test.targets = X_test.targets[idx]
test_loader = torch.utils.data.DataLoader(X_test, batch_size=1, shuffle=True)
class QuantumCircuit:
"""This class defines the quantum circuit structure and the run method which is used to calculate an expectation value"""
def __init__(self, qubit_count: int):
"""Define the quantum circuit in CUDA Quantum"""
kernel, thetas = cudaq.make_kernel(list)
self.kernel = kernel
self.theta = thetas
qubits = kernel.qalloc(qubit_count)
self.kernel.h(qubits)
# Variational gate parameters which are optimised during training
kernel.ry(thetas[0], qubits[0])
kernel.rx(thetas[1], qubits[0])
def run(self, thetas: torch.tensor) -> torch.tensor:
"""Excetute the quantum circuit to output an expectation value"""
expectation = torch.tensor(cudaq.observe(self.kernel, spin.z(0),
thetas).expectation_z(),
device=device)
return expectation
class QuantumFunction(Function):
"""Allows the quantum circuit to pass data through it and compute the gradients"""
@staticmethod
def forward(ctx, thetas: torch.tensor, quantum_circuit,
shift) -> torch.tensor:
# Save shift and quantum_circuit in context to use in backward
ctx.shift = shift
ctx.quantum_circuit = quantum_circuit
# Calculate exp_val
expectation_z = ctx.quantum_circuit.run(thetas)
ctx.save_for_backward(thetas, expectation_z)
return expectation_z
@staticmethod
def backward(ctx, grad_output):
"""Backward pass computation via finite difference parameter shift"""
thetas, expectation_z = ctx.saved_tensors
gradients = torch.zeros(len(thetas), device=device)
for i in range(len(thetas)):
shift_right = torch.clone(thetas)
shift_right[i] += ctx.shift
shift_left = torch.clone(thetas)
shift_left[i] -= ctx.shift
expectation_right = ctx.quantum_circuit.run(shift_right)
expectation_left = ctx.quantum_circuit.run(shift_left)
gradients[i] = 0.5 * (expectation_right - expectation_left)
return gradients * grad_output.float(), None, None
class QuantumLayer(nn.Module):
"""Encapsulates a quantum circuit and a quantum function into a quantum layer"""
def __init__(self, shift: torch.tensor):
super(QuantumLayer, self).__init__()
self.quantum_circuit = QuantumCircuit(1) # 1 qubit quantum circuit
self.shift = shift
def forward(self, input):
ans = QuantumFunction.apply(input, self.quantum_circuit, self.shift)
return ans
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# Neural network structure
self.conv1 = nn.Conv2d(1, 6, kernel_size=5)
self.conv2 = nn.Conv2d(6, 16, kernel_size=5)
self.dropout = nn.Dropout2d()
self.fc1 = nn.Linear(256, 64)
self.fc2 = nn.Linear(
64, 2
) # Output a 2D tensor since we have 2 variational parameters in our quantum circuit
self.hybrid = QuantumLayer(
torch.tensor(np.pi / 2)
) # Input is the magnitude of the parameter shifts to calculate gradients
def forward(self, x):
x = F.relu(self.conv1(x))
x = F.max_pool2d(x, 2)
x = F.relu(self.conv2(x))
x = F.max_pool2d(x, 2)
x = self.dropout(x)
x = x.view(1, -1)
x = F.relu(self.fc1(x))
x = self.fc2(x).reshape(
-1) # Reshapes required to satisfy input dimensions to CUDAQ
x = self.hybrid(x).reshape(-1)
return torch.cat((x, 1 - x), -1).unsqueeze(0)
# We move our model to the CUDA device to minimise data transfer between GPU and CPU
model = Net().to(device)
print(model)
optimizer = optim.Adam(model.parameters(), lr=0.001)
loss_func = nn.NLLLoss().to(device)
epochs = 20
epoch_loss = []
model.train()
for epoch in range(epochs):
batch_loss = 0.0
for batch_idx, (data, target) in enumerate(train_loader): # batch training
optimizer.zero_grad()
data, target = data.to(device), target.to(device)
# Forward pass
output = model(data).to(device)
# Calculating loss
loss = loss_func(output, target).to(device)
# Backward pass
loss.backward()
# Optimize the weights
optimizer.step()
batch_loss += loss.item()
epoch_loss.append(batch_loss / batch_idx)
print("Training [{:.0f}%]\tLoss: {:.4f}".format(
100.0 * (epoch + 1) / epochs, epoch_loss[-1]))
plt.plot(epoch_loss)
plt.title("Hybrid NN Training Convergence")
plt.xlabel("Training Iterations")
plt.ylabel("Neg Log Likelihood Loss")
# Testing on the test set
model.eval()
with torch.no_grad():
correct = 0
for batch_idx, (data, target) in enumerate(test_loader):
data, target = data.to(device), target.to(device)
output = model(data).to(device)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
loss = loss_func(output, target)
epoch_loss.append(loss.item())
print("Performance on test data:\n\tAccuracy: {:.1f}%".format(
correct / len(test_loader) * 100))
File added
# ARCHIVE Storage
The ARCHIVE storage is a complex solution expanding the platform for efficient distributed data complementing the existing IT4I storage systems.
It serves as the central data storage for archiving long-term data of laboratories and research projects,
particularly the REFRESH project (Research Excellence for Region Sustainability and High-tech Industries).
The ARCHIVE storage is built on [Nodeum][1] and [Quantum Scalar i6][2] technology.
## Technical Specifications
### Nodeum
Nodeum is a software for unstructured data management specifically designed to address issues related to data transfer, migration, search, and validation.
It provides Restful API, Web GUI and SSH interface
**Nodeum Archive Server specification** – Dell r760 (2x CPU Intel Xeon Silver 4510 2.4G, 12 Cores, 384 GB RAM, 2x SSD 960 GB, 2x 100 GbE ports QSFP28 including SR modules, 2x 1 GbE ports. iDrac Enterprise).
### Quantum Scalar i6
Quantum Scalar i6 is a modular tape drive offering a range of features
for easy data management and monitoring and a user-friendly console allowing users
to monitor performance, receive status notifications, etc.
**Quantum Scalar i6 specifications** - 575 LTO-9 slots, 4 x LTO-9 12Gb SAS Full Height drive, connected to the Dell r760 servers via 12 Gb SAS. **Total capacity of 10.35PB.**
**Disk Storage NVMe** – Infortrend EonStor GS 2000 (dual controller, 10 x U.3 NVMe SSD, PCIe Gen4, 7.68TB, 16 GB cache, 4 x 12 Gb SAS connection to the Dell r760 server). **Total capacity of 46TB.**
## Access Policy
To access the ARCHIVE storage, it is required to:
1. be a member of the REFRESH project,
1. get an approval from either [Tomas Kozubek][b] or [Jan Martinovic][c],
1. contact [support\[at\]it4i.cz][a] with a request to access.
[1]: https://www.nodeum.io/
[2]: https://www.quantum.com/en/products/tape-storage/scalar-i6/
[a]: mailto:support@it4i.cz
[b]: mailto:tomas.kozubek@vsb.cz
[c]: mailto:jan.martinovic@vsb.cz
# AWS CLI
You can use the AWS command line tools to issue commands or build scripts at your system's command line to perform AWS (including S3) tasks.
The AWS Command Line Interface (AWS CLI) provides commands for a broad set of AWS services. To get started, see the AWS Command Line Interface User Guide.
For more information, see the official site [https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#BasicsBucket][a].
## How to Load Modules:
To see loaded modules, use:
```
ml
```
To load modules, use:
```
ml awscli
```
To see module info, use:
```
ml spider AWS
```
For more information on how to work with modules, see
[https://docs.it4i.cz/software/modules/lmod/][1].
## Configuration
For configuration steps, follow the [e-INFRA CZ guide][b].
For the list of basic AWS CLI commands, see [https://docs.e-infra.cz/storage/object-storage/aws-cli/#controls-of-aws-cli---high-level-s3][e].
## Useful Links
* [Getting started][c]
* [Troubleshoot AWS CLI errors][f]
* [Configuration erros][d]
[1]: https://docs.it4i.cz/software/modules/lmod/
[a]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#BasicsBucket
[b]: https://docs.e-infra.cz/storage/object-storage/aws-cli/
[c]: https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html
[d]: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
[e]: https://docs.e-infra.cz/storage/object-storage/aws-cli/#controls-of-aws-cli---high-level-s3
[f]: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-troubleshooting.html
# CESNET S3 Storage
S3 service is a general service suited for most of the use cases. S3 service can be used for elementary data storing, automated backups, or various types of data handling applications. The service is supported by CESNET Storage Department: [du-support(at)cesnet.cz][email]
For more information, see [e-IINFRA CZ S3 Service documentation][a].
## Access:
To access to S3 you must:
* have an [e-Infra CZ account][3]
* be a [member of an active project][4]
### Steps
1. Fil in application at [https://einfra.cesnet.cz/allfed/registrar/?vo=VO_s3&group=s3_cl4&locale=en][b] to be a member of VO group. For more information about the S3 object storage, see [https://du.cesnet.cz/en/navody/object_storage/osobni_s3/start][f].
2. Once the application is approved you will be informed via email. Then please wait at least 30 minutes for our system to synchronize all data. Then you can continue to the Gatekeeper service to generate your credentials. To the [Gatekeeper service][g], you need to login via eduID.cz identity.
IT4I offers two tools for object storage management on Karolina and Barbora:
!!! Note
We recommend using the default versions installed.
* [S3cmd][1]
* [AWS CLI][2]
## Walkthroughs (Config Connection)
* [https://du.cesnet.cz/en/navody/object_storage/cesnet_s3/start][c]
* [https://www.s3express.com/kb/item26.htm][d]
[1]: s3cmd.md
[2]: awscli.md
[3]: https://docs.e-infra.cz/account/
[4]: https://docs.it4i.cz/general/access/project-access/
[a]: https://docs.e-infra.cz/storage/object-storage/s3-service/
[b]: https://einfra.cesnet.cz/allfed/registrar/?vo=VO_s3&group=s3_cl4&locale=en
[c]: https://du.cesnet.cz/cs/navody/object_storage/cesnet_s3/start
[d]: https://www.s3express.com/kb/item26.htm
[e]: https://filesender.cesnet.cz/
[f]: https://du.cesnet.cz/en/navody/object_storage/osobni_s3/start
[g]: https://access.du.cesnet.cz/#/access
[email]: mailto:du-support@cesnet.cz
# CESNET Data Storage
IT4Innovations' shared file systems are not intended as a backup for large amount of data or for long-term archiving purposes.
Academic staff and students of research institutions in the Czech Republic should use [CESNET Storage service][a].
CESNET data storage can be used by any organization or an individual person who is in the current employment relationship (employees) or the current study relationship (students) to a legal entity (organization) that meets the “Principles for access to CESNET Large infrastructure (Access Policy)”.
The user may only use the CESNET data storage for data transfer and storage associated with activities in science, research, development, spread of education, culture, and prosperity. For details, see “Acceptable Use Policy CESNET Large Infrastructure (Acceptable Use Policy, AUP)”.
The service is documented [here][b]. For special requirements contact directly CESNET Storage Department via e-mail [du-support(at)cesnet.cz][c].
The procedure to obtain the CESNET access is quick and simple.
## CESNET Storage Access
### Understanding CESNET Storage
!!! note
It is very important to understand the CESNET storage before uploading data. [Read][d] first.
Once registered for CESNET Storage, you may [access the storage][e] in number of ways. We recommend the SSHFS and RSYNC methods.
### SSHFS Access
!!! note
SSHFS: The storage will be mounted like a local hard drive
The SSHFS provides a very convenient way to access the CESNET Storage. The storage will be mounted onto a local directory, exposing the vast CESNET Storage as if it was a local removable hard drive. Files can be than copied in and out in a usual fashion.
First, create the mount point:
```console
$ mkdir cesnet
```
Mount the storage. Note that you can choose among ssh.du4.cesnet.cz (Ostrava) and ssh.du5.cesnet.cz (Jihlava).
Mount tier1_home **(only 5120M !)**:
```console
$ sshfs username@ssh.du4.cesnet.cz:. cesnet/
```
For easy future access, install your public key:
```console
$ cp .ssh/id_rsa.pub cesnet/.ssh/authorized_keys
```
Mount tier1_cache_tape for the Storage VO:
```console
$ sshfs username@ssh.du4.cesnet.cz:/cache_tape/VO_storage/home/username cesnet/
```
View the archive, copy the files and directories in and out:
```console
$ ls cesnet/
$ cp -a mydir cesnet/.
$ cp cesnet/myfile .
```
Once done, remember to unmount the storage:
```console
$ fusermount -u cesnet
```
### Rsync Access
!!! note
Rsync provides delta transfer for best performance and can resume interrupted transfers.
Rsync is a fast and extraordinarily versatile file copying tool. It is famous for its delta-transfer algorithm, which reduces the amount of data sent over the network by sending only the differences between the source files and the existing files in the destination. Rsync is widely used for backups and mirroring and as an improved copy command for everyday use.
Rsync finds files that need to be transferred using a "quick check" algorithm (by default) that looks for files that have changed in size or in last-modified time. Any changes in the other preserved attributes (as requested by options) are made on the destination file directly when the quick check indicates that the file's data does not need to be updated.
More about Rsync [here][f].
Transfer large files to/from CESNET storage, assuming membership in the Storage VO:
```console
$ rsync --progress datafile username@ssh.du4.cesnet.cz:VO_storage-cache_tape/.
$ rsync --progress username@ssh.du4.cesnet.cz:VO_storage-cache_tape/datafile .
```
Transfer large directories to/from CESNET storage, assuming membership in the Storage VO:
```console
$ rsync --progress -av datafolder username@ssh.du4.cesnet.cz:VO_storage-cache_tape/.
$ rsync --progress -av username@ssh.du4.cesnet.cz:VO_storage-cache_tape/datafolder .
```
Transfer rates of about 28MB/s can be expected.
[a]: https://du.cesnet.cz/
[b]: https://du.cesnet.cz/en/start
[c]: mailto:du-support@cesnet.cz
[d]: https://du.cesnet.cz/en/navody/home-migrace-plzen/start
[e]: https://du.cesnet.cz/en/navody/faq/start)
[f]: https://du.cesnet.cz/en/navody/rsync/start#pro_bezne_uzivatele
docs.it4i/storage/img/file-storage-block4.png

44.8 KiB

docs.it4i/storage/img/project-storage-overview2.png

36.6 KiB

docs.it4i/storage/img/project-storage-overview3.png

31.4 KiB

# NFSv4 File ACL
An NFSv4 ACL consists of one or more NFSv4 ACEs (Access Control Entry), each delimited by a comma or whitespace.
An NFSv4 ACE is written as a colon-delimited, 4-field string in the following format:
``` code
<type>:<flags>:<principal>:<permissions>
```
## ACE Elements
`<type>` - one of:
| Flag | Name |
| ---- | ----- |
| A | allow |
| D | deny |
| U | audit |
| L | alarm |
`<flags>` - zero or more (depending on `<type>`) of:
| Flag | Name |
| ---- | ------------------------------------------- |
| f | file-inherit |
| d | directory-inherit |
| p | no-propagate-inherit |
| i | inherit-only |
| S | successful-access |
| F | failed-access |
| g | group (denotes that <principal> is a group) |
`<principal>` - named user or group, or one of: `OWNER@`, `GROUP@`, `EVERYONE@`
`<permissions>` - one or more of:
| Flag | Name |
| ---- | --------------------------------- |
| r | read-data / list-directory |
| w | write-data / create-file |
| a | append-data / create-subdirectory |
| x | execute |
| d | delete |
| D | delete-child (directories only) |
| t | read-attrs |
| T | write-attrs |
| n | read-named-attrs |
| N | write-named-attrs |
| c | read-ACL |
| C | write-ACL |
| o | write-owner |
| y | synchronize |
## Example
``` code
[root@login2.karolina proj1]# nfs4_getfacl open-20-11
# file: open-20-11
A::OWNER@:rwaDxtTcCy
A::GROUP@:rxtcy
A:g:open-20-11@it4i.cz:rwaDxtcy
A::EVERYONE@:tcy
A:fdi:OWNER@:rwaDxtTcCy
A:fdi:GROUP@:rxtcy
A:fdig:open-20-11@it4i.cz:rwaDxtcy
A:fdi:EVERYONE@:tcy
```
# PROJ4 Object Storage
OpenStack Swift is a highly scalable, distributed object storage system
designed to store and retrieve large amounts of unstructured data.
It is an open-source project that provides a simple, scalable, and durable storage system for applications and services.
Swift is built to be highly available, fault-tolerant, and scalable, making it an ideal choice for storing large amounts of data.
Swift is designed to be highly modular, with a simple API that allows developers to easily integrate it into their applications.
It provides a RESTful API that can be accessed using a variety of programming languages,
making it easy to integrate with existing applications.
One of the key features of Swift is its ability to scale horizontally,
allowing it to handle large amounts of data and high levels of traffic.
It is also designed to be highly durable, with data being replicated across multiple nodes to ensure that it is always available.
Overall, OpenStack Swift is a powerful and flexible object storage system
that is well-suited for a wide range of applications and use cases.
## Accessing PROJ4
PROJ4 is accessible from all IT4Innovations clusters' login nodes
as well as from the outside.
Additionally, it allows to share data across clusters, etc.
User has to be part of a project, which is allowed to use S3 storage. If you haven't received your S3 credentials after your project was created, please send a request to support@it4i.cz asking for the "S3 PROJECT ACCESS", stating your IT4I login and name of your project (in the OPEN-XX-YY format or similar).
After that an active role on the S3 storage will be created and you will obtain via na email the credentials for using the S3 storage.
## How to Configure S3 Client
```console
$ sudo yum install s3cmd -y ## for debian based systems use apt-get
$ s3cmd --configure
.
.
.
.
Access Key: ***your_access***
Secret Key: ***your_secret_key***
Default Region: US
S3 Endpoint: obj.proj4.it4i.cz
DNS-style bucket+hostname:port template for accessing a bucket: obj.proj4.it4i.cz
Encryption password: RANDOM
Path to GPG program: /usr/bin/gpg
Use HTTPS protocol: True
HTTP Proxy server name:
HTTP Proxy server port: 0
.
.
.
Configuration saved to '/home/IT4USER/.s3cfg'
.
.
```
Please note, that the Encryption password should be defined by you instead of using the value "RANDOM".
Now you have to make some bucket for your data with **your_policy**, referencing your project (e.g. OPEN-XX-YY - if OPEN-XX-YY is your active and eligible project).
If you make a bucket without policy, we will not able to manage your project's data expiration and you might loose the data before the end of your actuall project - so please use the policy.
```console
~ s3cmd --add-header=X-Storage-Policy:OPEN-XX-YY mb s3://test-bucket
~ $ s3cmd put test.sh s3://test-bucket/
upload: 'test.sh' -> 's3://test-bucket/test.sh' [1 of 1]
1239 of 1239 100% in 0s 19.59 kB/s done
~ $ s3cmd ls
2023-10-17 13:00 s3://test-bucket
~ $ s3cmd ls s3://test-bucket
2023-10-17 13:09 1239 s3://test-bucket/test.sh
```
There is no possibility to set permissions for all members of a project,
so you have to set permissions for each user in a project.
Permission can be set only by the owner of the bucket.
```console
~ s3cmd setacl s3://test-bucket/test1.log --acl-grant=full\_control:user1
s3://test-bucket/test1.log: ACL updated
~ s3cmd setacl s3://test-bucket/test1.log --acl-grant=full\_control:user2
s3://test-bucket/test1.log: ACL updated
~ s3cmd setacl s3://test-bucket/test1.log --acl-grant=read:user3
s3://test-bucket/test1.log: ACL updated
~ s3cmd setacl s3://test-bucket/test1.log --acl-grant=write:user4
s3://test-bucket/test1.log: ACL updated
~ s3cmd info s3://test-bucket/test1.log
s3://test-bucket/test1.log (object):
File size: 1024000000
Last mod: Mon, 09 Oct 2023 08:06:12 GMT
MIME type: application/xml
Storage: STANDARD
MD5 sum: b5c667a723a10a3485a33263c4c2b978
SSE: none
Policy: none
CORS: none
ACL: OBJtest:user2: FULL\_CONTROL
ACL: \*anon\*: READ
ACL: user1: FULL\_CONTROL
ACL: user2: FULL\_CONTROL
ACL: user3: READ
ACL: user4: WRITE
URL: [http://195.113.250.1:8080/test-bucket/test1.log](http://195.113.250.1:8080/test-bucket/test1.log)
x-amz-meta-s3cmd-attrs: atime:1696588450/ctime:1696588452/gid:1001/gname:******/md5:******/mode:33204/mtime:1696588452/uid:******/uname:******
```
## Access to Multiple Projects
If a user needs to access multiple projects' data, it is needed to repeat the step asking the IT4I support for new credentials for the additional projects. In case you don't have the credentials assigned with the project activation, please send a request to support@it4i.cz.
As the first step, rename your current S3 configuration, so that it uniquely identifies your current project or organize it on your local storage accordingly.
```console
$ mv /home/IT4USER/.s3cfg /home/IT4USER/.s3cfg-OPEN-XX-YY
```
Then create new S3 configuration for the additional project (e.g. OPEN-AA-BB).
```console
$ s3cmd --configure
```
Rename or organize you newly created config.
```console
$ mv /home/IT4USER/.s3cfg /home/IT4USER/.s3cfg-OPEN-AA-BB
```
When acccessing the data of the different project specify the right configuration using the S3 commands.
```console
~ s3cmd -c /home/IT4USER/.s3cfg-OPEN-AA-BB --add-header=X-Storage-Policy:OPEN-AA-BB mb s3://test-bucket
~ $ s3cmd -c /home/IT4USER/.s3cfg-OPEN-AA-BB put test.sh s3://test-bucket/
upload: 'test.sh' -> 's3://test-bucket/test.sh' [1 of 1]
1239 of 1239 100% in 0s 19.59 kB/s done
~ $ s3cmd -c /home/IT4USER/.s3cfg-OPEN-AA-BB ls
2023-10-17 13:00 s3://test-bucket
~ $ s3cmd -c /home/IT4USER/.s3cfg-OPEN-AA-BB ls s3://test-bucket
2023-10-17 13:09 1239 s3://test-bucket/test.sh
```
## Bugs & Features
By default, the S3CMD client uses the so-called "multipart upload",
which means that it splits the uploaded file into "chunks" with a default size of 15 MB.
However, this upload method has major implications for the data capacity of the filesystem/fileset when overwriting existing files.
When overwriting an existing file in a "multipart" mode, the capacity is duplicated
(the file is not overwritten, but rewritten and the original file remains - but the capacity is allocated by both files).
This is a described swift bug for which there is no fix yet.
But there is a workaround and that is to disable "multipart upload" on the S3CMD client side.
```console
~ s3cmd --disable-multipart put /install/test1.log s3://test-bucket1
upload: '/install/test1.log' -> 's3://test-bucket1/test1.log' [1 of 1]
1024000000 of 1024000000 100% in 9s 99.90 MB/s done
```
This method is not recommended for large files, because it is not as fast and reliable as multipart upload, but it is the only way how to overwrite files without duplicating capacity.
# PROJECT Data Storage
The PROJECT data storage is a central storage for projects' and users' data on IT4Innovations.
The PROJECT data storage is accessible from all IT4Innovations clusters and allows to share data across clusters.
The storage is intended to be used throughout the whole project's lifecycle.
PROJECT data storage is not designed for computational tasks and must not be used directly for such purposes.
Please ensure that all files are copied to SCRATCH from the login nodes before submitting your job.
## Technical Overview
The PROJECT storage consists of three equal file storages (blocks) called PROJ1, PROJ2, and PROJ3.
Each file storage implements GPFS file system exported via NFS protocol using three NFS servers.
File storages provide high-availability and redundancy.
![](img/project-storage-overview3.png)
| Specification | Total | Per Block |
| ----------------- | -------------------|-------------------- |
| Protocol | NFS over GPFS |
| Total capacity | 15PB | 5PB |
| Throughput | 39GB/s | 13GB/s |
| IO Performance | 57kIOPS | 19kIOPS |
## Accessing PROJECT
All aspects of allocation, provisioning, accessing, and using the PROJECT storage are driven by project paradigm.
Storage allocation and access to the storage are based on projects (i.e. computing resources allocations) and project membership.
A project directory (actually implemented as an independent fileset) is created for every active project.
Default limits (quotas), default file permissions, and ACLs are set.
The project directory life cycle strictly follows the project's life cycle.
The project directory is removed after the project's data expiration.
### POSIX File Access
!!!note "Mountpoints"
PROJECT file storages are accessible at mountpoints `/mnt/proj1`, `/mnt/proj2`, and `/mnt/proj3`.
The PROJECT storage can be accessed via the following nodes:
| Cluster | Node(s) |
| ------------- | ----------------------------- |
| Karolina | Login, Compute, Visualization |
| Barbora | Login, Compute, Visualization |
To show the path to your project's directory on the PROJECT storage, use the `it4i-get-project-dir` command:
```console
$ it4i-get-project-dir OPEN-XX-XX
/mnt/proj3/open-XX-XX
```
### Project Quotas
The PROJECT storage enforces quotas on projects' usage (used capacity and allocated inodes).
Default quotas for capacity and amount of inodes per project are set by IT4Innovations.
| Project default quota | |
| --------------------- | ------- |
| Space quota | 20TB |
| Inodes quota | 5 mil. |
You can check the actual usage of the PROJECT storage (e.g. location of project directory, used capacity, allocated inodes, etc.) by executing the `it4ifsusage` command from the Login nodes' command line. The command lists all projects associated with the user.
```console
[vop999@login1.barbora ~]$ it4ifsusage
Quota Type Cluster / PID File System Space used Space limit Entries used Entries limit Last update
------------- --------------- ------------- ------------ ------------- -------------- --------------- -------------------
User barbora /home 11.1 MB 25.0 GB 122 500,000 2021-08-24 07:50:09
User karolina /home 354.6 MB 25.0 GB 3,194 500,000 2021-08-24 08:20:08
User barbora /scratch 256.5 GB 10.0 TB 169 10,000,000 2021-08-24 07:50:19
User karolina /scratch 52.5 GB 100.0 TB 967 20,000,000 2021-08-24 08:20:18
Project open-XX-XX proj1 3.9 TB 20.0 TB 212,377 5,000,000 2021-08-24 08:20:02
Project open-YY-YY proj3 9.5 MB 20.0 TB 182 5,000,000 2021-08-24 08:20:02
Project open-ZZ-ZZ proj2 844.4 GB 20.0 TB 797 5,000,000 2021-08-24 08:20:02
```
The information can also be found in IT4Innovations' [SCS information system][b].
!!!note
At this time, only PIs can see the quotas of their respective projects in IT4Innovations' SCS information system.
We are working on making this information available to all users assigned to their projects.
#### Increasing Project Quotas
It is preferred that you request additional storage space allocation in advance in your application for computational resources.
Alternatively, if the project is already active, contact [IT4I support][a].
### ACL and File Permissions
Access to a project directory and containing files is restricted by Unix file permissions and file access control lists (ACLs).
Default file permissions and ACLs are set by IT4Innovations during project directory provisioning.
## Backup and Safety
!!!important "Data Backup"
Data on the PROJECT storage is **not** backed up.
The PROJECT storage utilizes fully redundant design, redundant devices, highly available services, data redundancy, and snapshots. For increased data protection, disks in each disk array are connected in Distributed RAID6 with two hot-spare disks, meaning the disk array can recover full redundancy after two simultaneous disk failures.
However, the storage does not provide data backup, so we strongly recommend using the [CESNET storage][1] for making independent copies of your data.
### Snapshots
The PROJECT storage provides snapshot functionality. A snapshot represents a state of a filesystem at a particular point in time. Snapshots are created for all projects on fileset (i.e. project directory) level.
Snapshots are created every day, snapshots older than seven days are deleted.
Snapshots are not calculated towards the disk quota.
Files in snapshots are accessible directly by users in the special subdirectory of each project directory named `.snapshots`.
Snapshots are read-only.
Snapshots' names have the `YYYY-MM-DD-hhmmss` format.
```console
[vop999@login1.karolina ~]# ls -al /mnt/proj3/open-XX-XX/.snapshots
total 4
dr-xr-xr-x. 2 root root 4096 led 14 12:14 .
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 ..
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-01-022441
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-02-022544
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-03-022949
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-04-023454
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-05-024152
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-06-020412
drwxrws---. 16 vop999 open-XX-XX 4096 led 20 16:36 2021-03-07-021446
```
<! --- (HA data replication?) -->
<! --- (balancing in case of overload (data migration?) -->
## Computing on PROJECT
!!!important "I/O Jobs"
Stage files for I/O calculations onto the SCRATCH storage.
PROJECT storage is not designed for computational tasks and must not be used directly for such purposes.
Please ensure that all files are copied to SCRATCH from the login nodes before submitting your job.
<! --- See also: data storage policy on filesystems (link?) -->
<! --- ## Technical Specification -->
<! --- For a detailed technical specification, see the Technical Specification section. -->
## Summary
| PROJECT Storage | |
| -------------------- | ------------------- |
| Mountpoint | /mnt/proj{1,2,3} |
| Capacity | 15PB |
| Throughput | 39GB/s |
| IO Performance | 57kIOPS |
| Default project space quota | 20TB |
| Default project inodes quota | 5 mil. |
[1]: ../storage/cesnet-storage.md
[a]: mailto:support@it4i.cz
[b]: https://scs.it4i.cz/projects
# s3cmd
S3cmd is a command-line tool for managing data in Amazon Simple Storage Service (S3). It is a free and open-source tool that is available for Linux, macOS, and Windows.
For more information, see [https://docs.e-infra.cz/storage/object-storage/s3cmd/][a].
We recommend using the S3cmd tool already installed on Karolina.
## How to Load Modules:
To see loaded modules, use:
```
ml
```
To load modules, use:
```
ml s3cmd
```
To see module info, use:
```
ml spider s3cmd
```
For more information on how to work with modules, see
[https://docs.it4i.cz/software/modules/lmod/][1].
## Configuration
The s3cmd configuration file is named `.s3cfg` and it is located in the user's home directory, e.g. `/home/username/($HOME)`.
The configuration file has to be created first:
```
s3cmd --configure
```
For configuration steps, follow the [e-INFRA CZ guide][b].
For the list of basic S3cmd commands, see [https://docs.e-infra.cz/storage/object-storage/s3cmd/#basic-s3cmd-commands][c].
## FAQ
Error when creating bucket:
```
ERROR: S3 error: 400 (InvalidLocationConstraint): The specified location-constraint is not valid
```
Solution: set location:
```
s3cmd --bucket-location=":default-placement" mb s3://newbucketname
```
## Useful Links
* [Introduction][d]
* [Usage][e]
* [Common S3cmd Errors][g]
* [Configuration erros][f]
[1]: https://docs.it4i.cz/software/modules/lmod/
[a]: https://docs.e-infra.cz/storage/object-storage/s3cmd/
[b]: https://docs.e-infra.cz/storage/object-storage/s3cmd/#configuration-of-s3cmd-tool
[c]: https://docs.e-infra.cz/storage/object-storage/s3cmd/#basic-s3cmd-commands
[d]: https://s3tools.org/s3cmd
[e]: https://s3tools.org/usage
[f]: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
[g]: https://s3tools.org/kb/category1.htm
# Standard File ACL
Access control list (ACL) provides an additional, more flexible permission mechanism for file systems. It is designed to assist with UNIX file permissions. ACL allows you to give permissions for any user or group to any disk resource.
## Show ACL
To show permissions, use:
```code
$ getfacl <file/dir>
```
### Examples
Set all permissions for user john to file named testfile:
```code
$ setfacl -m "u:john:rwx" testfile
```
Check permissions:
```code
$ getfacl testfile
# file: testfile
# owner: someone
# group: someone
user::rw-
user:john:rwx
group::r--
mask::rwx
other::r--
```
Change permissions for user john:
```code
$ setfacl -m "u:john:r-x" testfile
```
Check permissions:
```code
$ getfacl testfile
# file: testfile
# owner: someone
# group: someone
user::rw-
user:john:r-x
group::r--
mask::r-x
other::r--
```
Remove all ACL entries:
```code
$ setfacl -b testfile
```
Check permissions:
```code
$ getfacl testfile
# file: testfile
# owner: someone
# group: someone
user::rw-
group::r--
other::r--
```
## Output of LS Command
You will notice that there is an ACL for a given file because it will exhibit `+` after its Unix permissions in the output of `ls -l`.
```code
$ ls -l testfile
crw-rw----+ 1 someone someone 14, 4 nov. 9 12:49 testfile
```
## Modify ACL
The ACL can be modified using the `setfacl` command.
You can list file/directory permission changes without modifying the permissions (i.e. dry-run) by appending the `--test` flag.
To apply operations to all files and directories recursively, append the `-R/--recursive` argument.
To set permissions for a user (user is either the user name or ID):
```code
$ setfacl -m "u:user:permissions" <file/dir>
```
To set permissions for a group (group is either the group name or ID):
```code
$ setfacl -m "g:group:permissions" <file/dir>
```
To set permissions for others:
```code
$ setfacl -m "other:permissions" <file/dir>
```
To allow all newly created files or directories to inherit entries from the parent directory (this will not affect files which will be copied into the directory):
```code
$ setfacl -dm "entry" <dir>
```
To remove a specific entry:
```code
$ setfacl -x "entry" <file/dir>
```
To remove the default entries:
```code
$ setfacl -k <file/dir>
```
To remove all entries (entries of the owner, group and others are retained):
```code
$ setfacl -b <file/dir>
```
Source: [wiki.archlinux.org][1]
[1]: https://wiki.archlinux.org/title/Access_Control_Lists
# VLQ
[The VLQ quantum computer][a], a joint collaboration between EuroHPC JU and the LUMI-Q Consortium,
is hosted at IT4innovations and represents a significant advancement in European quantum computing infrastructure.
![](../img/IQM-Czech-PR.png)
Equipped with 24 superconducting transmon qubits arranged in a unique star-shaped topology,
the VLQ architecture significantly reduces the need for swap operations—a common bottleneck in many quantum circuits—thereby optimizing the utilization of coherence time and enhancing computational efficiency.
This innovative design allows for more reliable, coherent quantum calculations and experiments.
![](../img/IQMStar24.png)
The system’s user-friendly accessibility is highlighted by its compatibility
with widely used quantum programming frameworks such as Qiskit and CIRQ,
enabling seamless integration into existing quantum research workflows.
Additionally, researchers can leverage the power of the integrated Karolina supercomputer to offload more complex hybrid quantum-classical tasks.
[a]: https://www.it4i.cz/en/infrastructure/vlq-quantum-computer
\ No newline at end of file
{% extends "main.html" %}
{% block content %}
<h1>404 - Not found</h1>
{% endblock %}