Home‎ > ‎Server config‎ > ‎

Building the Bioinformatics Petabyte JBOD ZFS filesystem

Documentation & notes for the construction process of a petabyte+ ZFS filesystem for replication:

REMEMBER TO DISABLE READ CACHE ON THE RAID CONTROLLER!!
REMEMBER TO CREATE MAINTENANCE CRON JOBS!!!!

First, run: 
udevadm trigger
to make sure all the scsi devices are freshly probed, and /dev/disk/by-id is up to date.

then, construct the device candidate file:

ls -al /dev/disk/by-id|grep scsi|grep -v part|awk '{ print $9 }'>jbod_scsi_ids

This file will contain only the JBOD drives' SCSI ID's, which can be referenced in a script by /dev/disk/by-id/<line>.
The last 6 devices were moved to a separate file for hotspares.

Topology:

270 x 6TB SATA drives are available to be pooled. We decided to create a 33 raidz1 pools with 7+1 drives each, and 6 hotspares shared. A single mount point named /backup is aliased for this pool. Compression will be enabled.

2 mirrored SSD's are enabled for write caching, and one SSD for read cache.

Now build a C program to construct the pools:
####################################################
#define _GNU_SOURCE
#include <stdlib.h>
#include <stdio.h>
#include <strings.h>
#include <fcntl.h>
#include <errno.h>
#include <sys/types.h>
#include <unistd.h>

#define BUF_SIZE 100
#define COMMAND_SIZE 5000
#define GROUPS 33
#define DISKS 8
#define INPUTFILE "jbod_scsi_ids"
#define HOTSPARES "jbod_hotspares"
#define NUM_HOTSPARES 6
#define COMMAND "/usr/sbin/zpool add pool0 raidz1 "
#define PATH "/dev/disk/by-id/"
#define COMMAND_HOTSPARES "/usr/sbin/zpool add pool%u spare "

void remove_new_line(char *);

int main(void) {

  unsigned char i,j = 0;
  unsigned int groups = 0;
  unsigned int disks = 0;
  FILE*  fd;
  char buffer[5000];
  char path[1000];
  char command[COMMAND_SIZE];

  fd = fopen(INPUTFILE, "r");
  sprintf(path,PATH);

/* Create the base pools from the block devices - first one is created from 1st 8 devices, then these are added to pool0 */
/* Data files were edited by hand to remove hotspares and first 8 */

 printf("\n\nADDING BASE POOLS:\n\n");
 for(i=1;i<GROUPS;i++) {
       sprintf(command,COMMAND,i);
       for(j=0;j<DISKS;j++) {
          fgets(buffer, 100, fd);
          strcat(command," ");
          remove_new_line(buffer);
          strcat(command,path);
          strcat(command,buffer);
   }
   printf("\n\n%s\n",command);
   system(command);
 }

 fclose (fd);

  /* Now add hotspares shared on all pools */
  printf("\n\nADDING HOTSPARES TO THE POOLS:\n\n");
  fd = fopen(HOTSPARES, "r");

  for(i=0;i<GROUPS;i++) {
        sprintf(command,COMMAND_HOTSPARES,i);
        for(j=0;j<NUM_HOTSPARES;j++) {
              fgets(buffer, 100, fd);
              strcat(command," ");
              remove_new_line(buffer);
              strcat(command,path);
              strcat(command,buffer);
        }
        rewind(fd);
        printf("\n\n%s\n",command);
        system(command);
  }

  fclose (fd);

 /* Enable compression */
printf("\n\nEnabling compression:\n");
system("/usr/sbin/zfs set compression=lz4 pool0");

  /* Now add caches */
  printf("\n\nREMEMBER TO ADD CACHES MANUALLY!!!!!!!\n");
  printf("Something like:\n");
  printf("zpool add backup log mirror /dev/disk/by-id/<SSD1> /dev/disk/by-id/<SSD2> cache /dev/disk/by-id/<SSD3>\n");
}


/* Auxilliary functions */

void remove_new_line(char* string) {
  size_t length;
  if( (length =strlen(string) ) >0) {
          if(string[length-1] == '\n')
              string[length-1] ='\0';
  }
}
####################################################

Turn compression on:

zfs set compression=lz4 pool0


Enable caches:

zpool add pool0 log mirror /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV652301LR150MGN /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV717300EN150MGN cache /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV71720B5C150MGN

Alias mountpoint:

zfs umount pool0
zfs set mountpoint=/backup pool0
zfs mount pool0

Then get status:

zpool  status

Add cronjob to ensure ZFS startup on boot:

crontab -e
@reboot    /usr/sbin/zfs mount pool0



All should be good!