Documentation & notes for the construction process of a petabyte+ ZFS filesystem for replication: REMEMBER TO DISABLE READ CACHE ON THE RAID CONTROLLER!! REMEMBER TO CREATE MAINTENANCE CRON JOBS!!!! First, run: udevadm trigger to make sure all the scsi devices are freshly probed, and /dev/disk/by-id is up to date. then, construct the device candidate file: ls -al /dev/disk/by-id|grep scsi|grep -v part|awk '{ print $9 }'>jbod_scsi_ids This file will contain only the JBOD drives' SCSI ID's, which can be referenced in a script by /dev/disk/by-id/<line>. The last 6 devices were moved to a separate file for hotspares. Topology: 270 x 6TB SATA drives are available to be pooled. We decided to create a 33 raidz1 pools with 7+1 drives each, and 6 hotspares shared. A single mount point named /backup is aliased for this pool. Compression will be enabled. 2 mirrored SSD's are enabled for write caching, and one SSD for read cache. Now build a C program to construct the pools: #################################################### #define _GNU_SOURCE #include <stdlib.h> #include <stdio.h> #include <strings.h> #include <fcntl.h> #include <errno.h> #include <sys/types.h> #include <unistd.h> #define BUF_SIZE 100 #define COMMAND_SIZE 5000 #define GROUPS 33 #define DISKS 8 #define INPUTFILE "jbod_scsi_ids" #define HOTSPARES "jbod_hotspares" #define NUM_HOTSPARES 6 #define COMMAND "/usr/sbin/zpool add pool0 raidz1 " #define PATH "/dev/disk/by-id/" #define COMMAND_HOTSPARES "/usr/sbin/zpool add pool%u spare " void remove_new_line(char *); int main(void) {  unsigned char i,j = 0;  unsigned int groups = 0;  unsigned int disks = 0;  FILE* fd;  char buffer[5000];  char path[1000];  char command[COMMAND_SIZE];  fd = fopen(INPUTFILE, "r");  sprintf(path,PATH); /* Create the base pools from the block devices - first one is created from 1st 8 devices, then these are added to pool0 */ /* Data files were edited by hand to remove hotspares and first 8 */  printf("\n\nADDING BASE POOLS:\n\n");  for(i=1;i<GROUPS;i++) {      sprintf(command,COMMAND,i);      for(j=0;j<DISKS;j++) {       fgets(buffer, 100, fd);       strcat(command," ");       remove_new_line(buffer);       strcat(command,path);       strcat(command,buffer);   }   printf("\n\n%s\n",command);   system(command);  }  fclose (fd);  /* Now add hotspares shared on all pools */  printf("\n\nADDING HOTSPARES TO THE POOLS:\n\n");  fd = fopen(HOTSPARES, "r");  for(i=0;i<GROUPS;i++) {      sprintf(command,COMMAND_HOTSPARES,i);      for(j=0;j<NUM_HOTSPARES;j++) {          fgets(buffer, 100, fd);          strcat(command," ");          remove_new_line(buffer);          strcat(command,path);          strcat(command,buffer);      }      rewind(fd);      printf("\n\n%s\n",command);      system(command);  }  fclose (fd);  /* Enable compression */ printf("\n\nEnabling compression:\n"); system("/usr/sbin/zfs set compression=lz4 pool0");  /* Now add caches */  printf("\n\nREMEMBER TO ADD CACHES MANUALLY!!!!!!!\n");  printf("Something like:\n");  printf("zpool add backup log mirror /dev/disk/by-id/<SSD1> /dev/disk/by-id/<SSD2> cache /dev/disk/by-id/<SSD3>\n"); } /* Auxilliary functions */ void remove_new_line(char* string) {  size_t length;  if( (length =strlen(string) ) >0) {       if(string[length-1] == '\n')           string[length-1] ='\0';  } } #################################################### Turn compression on: zfs set compression=lz4 pool0 Enable caches: zpool add pool0 log mirror /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV652301LR150MGN /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV717300EN150MGN cache /dev/disk/by-id/ata-INTEL_SSDSC2BB150G7_PHDV71720B5C150MGN Alias mountpoint: zfs umount pool0 zfs set mountpoint=/backup pool0 zfs mount pool0 Then get status: zpool status Add cronjob to ensure ZFS startup on boot: crontab -e @reboot   /usr/sbin/zfs mount pool0 All should be good! |
Home‎ > ‎Server config‎ > ‎