import { addBanner, addArticle, addChapterTitle, addHeader, addParagraph, addSubHeader } from "/scripts/article.js";
import { addInset, addInsetList, addInsetCodeListing, addInsetBulletList } from "/scripts/inset.js";
import { addImageWithCaption, addButtonGroup } from "/scripts/visuals.js";
import { menu } from "/scripts/buttongroups.js";
const main = document.querySelector("main");
main.append(addBanner("Linux Storage Systems", "Kevin Dankwardt", "Disk Partitions, Formatting and Mounting"))
main.append(addArticle())
const article = document.querySelector("article")
article.append(addHeader("Basic Commands for Storage Partitions"))
article.append(addParagraph("A simple command to list partitions is"))
article.append(addInset("cat /proc/partitions"))
article.append(addParagraph("Output will typically look something like this"))
article.append(addInsetCodeListing(["1. major minor #blocks name", "2.", "3. 1 0 4096 ram0", "4. 1 1 4096 ram1", "5. 1 2 4096 ram2", "6. 1 3 4096 ram3", "7. 1 4 4096 ram4", "8. 1 5 4096 ram5", "9. 1 6 4096 ram6", "10. 1 7 4096 ram7", "11. 1 8 4096 ram8", "12. 1 9 4096 ram9", "13. 1 10 4096 ram10", "14. 1 11 4096 ram11", "15. 1 12 4096 ram12", "16. 1 13 4096 ram13", "17. 1 14 4096 ram14", "18. 1 15 4096 ram15", "19. 179 0 31166976 mmcblk0", "20. 179 1 262144 mmcblk0p1", "21. 179 2 30900736 mmcblk0p2", "22. 8 0 4883770583 sda", "23. 8 1 131072 sda1", "24. 8 2 4883638272 sda2"]))
article.append(addParagraph("Note that proc is a pseudo-file system providing an interface to the kernel data structures so when used as shown in figure 1, we are getting the data shown in the output from the kernel. The major number shown tells us which driver is being used (for example, 8 represents SCSI) and the minor number tells us which disk and which controller the partition is on."))
article.append(addParagraph("We can also get a list of partitions with the command"))
article.append(addInset("sudo fdisk -l"))
article.append(addParagraph("The output from this command is shown in listing 1."))
article.append(addParagraph("If we want to look at a specific disk, we can do that with a command like"))
article.append(addInset("sudo fdisk /dev/sda2"))
article.append(addParagraph("Output from this command will look like that shown below."))
article.append(addInsetCodeListing([">Welcome to fdisk (util-linux 2.33.1).", "Changes will remain in memory only, until you decide to write them.", "Be careful before using the write command.", "", "The size of this disk is 4.6 TiB (5000845590528 bytes). DOS partition table format cannot be used on drives for volumes larger than 4294966784 bytes for 512-byte sectors. Use GUID partition table format (GPT).", "", "Command (m for help):"]))
article.append(addParagraph("Note that what we are seeing here is an interface that allows us to type in a command for fdisk (such as m for help). We can also type p to see a list of partitions on that disk and typical output from this is shown below"))
article.append(addInsetCodeListing([">Welcome to fdisk (util-linux 2.33.1).", "Changes will remain in memory only, until you decide to write them.", "Be careful before using the write command.", "", "The size of this disk is 4.6 TiB (5000845590528 bytes). DOS partition table format cannot be used on drives for volumes larger than 4294966784 bytes for 512-byte sectors. Use GUID partition table format (GPT).", "", "Command (m for help):", "", "Disk /dev/sda2: 4.6 TiB, 5000845590528 bytes, 9767276544 sectors", "Units: sectors of 1 * 512 = 512 bytes", "Sector size (logical/physical): 512 bytes / 4096 bytes", "I/O size (minimum/optimal): 4096 bytes / 4096 bytes", "Disklabel type: dos", "Disk identifier: 0x73736572", "", "Device Boot Start End Sectors Size Id Type", "/dev/sda2p1 1920221984 3736432267 1816210284 866G 72 unknown", "/dev/sda2p2 1936028192 3889681299 1953653108 931.6G 6c unknown", "/dev/sda2p3 0 0 0 0B 0 Empty", "/dev/sda2p4 27722122 27722568 447 223.5K 0 Empty" , "", "Partition 4 does not start on physical sector boundary.", "", "Partition table entries are not in disk order.", "", "Command (m for help):"]))
article.append(addParagraph("You can use fdisk to manage these partitions, that is they can be created, deleted, given a type and so on and we can also use find partitions based on that type. Along with parted, fdisk is one of the most popular partitioning tools and we will look at managing partitions next."))
article.append(addParagraph("Note, in fdisk, q will quit back to the command prompt."))
article.append(addHeader("Create, Mount and Unmount File Systems"))
article.append(addSubHeader("CREATE"))
article.append(addParagraph("In partitioning a disk, we are setting aside some space for it, but for it to be usable by Linux, it needs to be partitioned. A typical format command would be"))
article.append(addInset("mkfs -t "))
article.append(addParagraph("Typically, type will be ext4 which is quite common, but there are other types and the partition is obviously the partition you want to format. Be very careful with this and always make sure you have the right partition since formatting the wrong partition could be very bad!"))
article.append(addParagraph("As an example, we might want to format the first partition on the second disk so the command would be"))
article.append(addInset("mkfs -t ext4 /dev/sdb1"))
article.append(addParagraph("Although it is unusual, you can format and mount an ordinary file which would fill it with binary zeros."))
article.append(addSubHeader("MOUNT"))
article.append(addParagraph("The process of mounting involves creating an association between a partition and a directory. You would normally only mount a partition to an empty directory although it doesn't have to be, but anything already in the directory when you mount as partition to it will be hidden."))
article.append(addParagraph("A typical mount command would look like this."))
article.append(addInset("mount /dev/sdb1 /myusbdisk"))
article.append(addParagraph("We could also specify the type here."))
article.append(addInset("mount -t ext4 /dev/sdb1 /myusbdisk"))
article.append(addParagraph("but Linux is usually able to infer this."))
article.append(addParagraph("There are also some special file systems that can be mounted although they are not disk partitions. When mounting one of these, you do need to specify the type and rather than specify a partition, you would provide the name of the filesystem which is usually the same as the type. For example, the proc filesystem is normally mounted to the proc directory, so the mount command would look like this."))
article.append(addInset("mount -t proc proc /proc"))
article.append(addParagraph("Similarly, the command to mount the sysfile system is"))
article.append(addInset("mount -t sysfile sysfile /sys"))
article.append(addParagraph("These are filesystems that do not correspond with a specific disk space."))
article.append(addParagraph("If you are mounting an NFS (Network File Share) partition, the syntax is again a little different in that rather than specifying a partition in the usual way, we are specifying a server name and the directory on that server that you want to mount."))
article.append(addInset("mount myserver:my dir /nfs-dir"))
article.append(addParagraph("Again, we can specify the type which is nfs"))
article.append(addInset("mount -t nfs myserver:my dir /nfs-dir"))
article.append(addParagraph("but this is not necessary."))
article.append(addParagraph("We can also mount the debug file system in this way and that it typically mounted to /sys/kernel/debug."))
article.append(addInset("mount -t debugfs debugfs /sys/kernel/debug"))
article.append(addParagraph("There are a number of options with mount and you can see these on the man page or on the online man page under Command Line Options."))
article.append(addParagraph("Note - the loop option used with o: allows you to mount to a directory, a fileststem that is in a file that is already in a mounted file system so it can be useful for mounting a file such as an iso file which you can then access theough the directory you have mounted the file to."))
article.append(addParagraph("Remount can be useful if you want to mount a filesystem using different options. For example, you might want to remount a filesystem as read only"))
article.append(addInset("mount o remount,ro \/myusbdisk"))
article.append(addParagraph("There are also special options for different types of file system. The internet has plenty of info relating to mount - the man page also gives some detail regarding available options."))
article.append(addSubHeader("UNMOUNT"))
article.append(addParagraph("You can unmount a filesystem using the umount command (note that there is no n in the command itself), provided that filesystem is not currently in use. In use can mean that you have a file open or a process is using a file in that filesystem. It can also mean that some process has a directory in that filesystem as its current directory. For instance, if you cd to the directory that a usb stick is mounted to, that will prevent you from unmounting it. Before you can unmount, you may need to ensure all files are closed, use cd to change to a directory not on the filesystem or, if some process other than a shell is using a file on that filesystem, you may need to kill it."))
article.append(addParagraph("When you unmount a disk, it is synced so any pending io operations are finished. It is useful to know that Linux will typically be configured to wait up to 30 seconds before writing something so if you have just written a file to a usb stick, you may need to wait for a bit before the write operation is performed."))
article.append(addParagraph("You can also do a lazy unmount using the -l option."))
article.append(addInset("umount -l"))
article.append(addParagraph("This immediately detaches the mount from the file hierarchy so that will prevent any process from accessing it to open a file, navigate to a directory and so on, but it doesn't completely unmount the filesystem until all processes using it have finished. The disk is not fully unmounted until all operations accessing that disk have been completed but the main thing is that it will prevent any other processing from accessing the disk so it is also not fully mounted."))
article.append(addParagraph("It is possible to force an unmount using the -f option. This can be useful if you have, for instance, a hung server."))
article.append(addHeader("Putting It All Into Practice"))
article.append(addParagraph("For practice purposes, I have an 8GB USB stick formatted with NTFS and I am going to insert this into my Raspberry Pi in order to perform some basic tasks with it. Before we start, I want to see what partitions we already have so I will check that with"))
article.append(addInset("sudo fdisk -l"))
article.append(addParagraph("This gives us a list which includes the following:"))
article.append(addInsetCodeListing(["Disk /dev/mmcblk0: 119.08 GiB, 127865454592 bytes, 249737216 sectors", "Units: sectors of 1 * 512 = 512 bytes", "Sector size (logical/physical): 512 bytes / 512 bytes", "I/O size (minimum/optimal): 512 bytes / 512 bytes", "Disklabel type: dos", "Disk identifier: 0x64538962", "","Device Boot Start End Sectors Size Id Type", "/dev/mmcblk0p1 8192 532479 524288 256M c W95 FAT32 (LBA)", "/dev/mmcblk0p2 532480 249737215 249204736 118.8G 83 Linux", "", "Disk /dev/sda: 4.55 TiB, 5000981077504 bytes, 9767541167 sectors", "Disk model: Portable", "Units: sectors of 1 * 512 = 512 bytes", "Sector size (logical/physical): 512 bytes / 4096 bytes", "I/O size (minimum/optimal): 4096 bytes / 4096 bytes", "Disklabel type: gpt", "Disk identifier: C38D37C5-915A-464A-9404-217E98AA74B6", "", "Device Start End Sectors Size Type", "/dev/sda1 34 262177 262144 128M Microsoft reserved", "/dev/sda2 264192 9767540735 9767276544 4.5T Microsoft basic data", "", "Partition 1 does not start on physical sector boundary.", "", "Disk /dev/sdb: 232.89 GiB, 250059350016 bytes, 488397168 sectors", "Disk model:", "Units: sectors of 1 * 512 = 512 bytes", "Sector size (logical/physical): 512 bytes / 4096 bytes", "I/O size (minimum/optimal): 4096 bytes / 4096 bytes", "Disklabel type: dos", "Disk identifier: 0x64538962", "", "Device Boot Start End Sectors Size Id Type", "/dev/sdb1 8192 532479 524288 256M c W95 FAT32 (LBA)", "/dev/sdb2 532480 488397167 487864688 232.6G 83 Linux"]))
article.append(addParagraph("There are three disks attached to the Raspberry Pi (I am using the term loosely, here one of those disks is the MicroSD card that holds the OS). These are as follows:"))
article.append(addParagraph("Missing table!"))
article.append(addParagraph("Knowing what disks are attached and knowing the size of the disk you are attaching is important in allowing you to identify the correct disk. When the usbstick is inserted into a usb port on the Raspberry Pi, it is automatically mounted so the device is sdc, the mount point is /media/philip/UUI and the size is shown as 7.5GB. We want to format the disk but we can't do that whilst it is mounted so we will unmount it with"))
article.append(addInset("umount /media/philip/UUI"))
article.append(addParagraph("I want too format this usb stick with the ext4 file system."))
article.append(addInset("sudo mkfs -t ext4 /dev/sdc1"))
article.append(addParagraph("With that done, I now want to mount this drive to a folder in my home drive called usb stick."))
article.append(addInset("sudo mount /dev/sdc1 ~/usbstick"))
article.append(addParagraph("There are a couple of methods we can use to verify that the device was mounted as expected. The first is with the command"))
article.append(addInset("mount"))
article.append(addParagraph("which lists all mount points. The second is to simply cd to the directory we mounted the usb stick to. Remember that we just formatted it so this method won't be particularly helpful at this point."))
article.append(addParagraph("We can unmount the disk with the command"))
article.append(addInset("sudo umount /home/philip/usbstick"))
article.append(addParagraph("You may have read or heard that some people do not like to refer to files that begin with a dot as being hidden files. They are hidden from a basic ls command, but there is no difficulty in seeing them if you want to simply by adding an option to the ls command (-a). In contrast, hidden files in an OS like Windows are more genuinely hidden because if you haven't see these to be visible, there is no option that allows you to view them. I will come back to this point in a second. It was mentioned earlier that if you mount a disk into a directory that already contains some files, these will be hidden. Let's find out how hidden they are."))
article.append(addParagraph("The usb stick has been unmounted but, of course, our home drive still contains the directory (usbstick) we created as a mount point for the usb stick. Now, I have navigate into that directory and created a file there. The type of file/name etc is unimportant, the point being that we now have a file in that directory so if we mount the usb stick back to that directory, the file will be hidden. As a matter of interest, the name of the file I created is bananaicecream.txt"))
article.append(addParagraph("If we mount the drive back to the same mount point, we can now get a directlry listing and we will see a list of the files on that usb stick, but we don't see the bananaicecream.txt file (which is now hidden). Next, we will try adding the -a option to our ls command and we still don't see the bananaicecream.txt file. This is because it is genuinely hidden rather than simply being inaccessible without the -a option. In this context, it is probably easier to understand why people may be reluctant to refer to files that start with a . in Linux as being hidden."))
article.append(addHeader("Use Block Device Attributes"))
article.append(addParagraph("Identifying storage is very important for a number of reasons. If you are doing anything with a device including unmounting it, mounting it or formatting it, you want to be sure (especially when formatting it) that you are working with the correct device. The used space device daemon collects information on any detectted disks and this includes dynamically detected disks, for example if you insert a usb stick. This generates files in a series of directories in the /dev/disk folder. The files are sorted into subdirectories depending on what information is recorded and these subdirectories are:"))
article.append(addInsetCodeListing(["drwxr-xr-x 2 root root 260 Jan 25 19:54 by-id", "drwxr-xr-x 2 root root 100 Jan 25 20:06 by-label", "drwxr-xr-x 2 root root 80 Jan 22 12:32 by-partlabel", "drwxr-xr-x 2 root root 140 Jan 25 19:54 by-partuuid", "drwxr-xr-x 2 root root 260 Jan 25 19:54 by-path", "drwxr-xr-x 2 root root 120 Jan 25 20:06 by-uuid"]))
article.append(addParagraph("In each directory, there is a small file for each partition or device and these are listed by id, label, partlabel and so on. If we want to see these listed by uuid we can cd to the by-uuid directory and list them. Note that there is a uuid and a partuuid directory. The difference between these is that the uuid directory will only list formatted partitions. A partition has a uuid before it is partitioned and one after."))
article.append(addParagraph("The partuuid directory will also show any unformatted partitions which means that we are able to get some information on a partition without having to format it first."))
article.append(addParagraph("The commands blkid and lsblk also provide some useful information, particularly lsblk which provides information such as mount points. If I run the lsblk command on my Raspberry Pi, the results are as follows:"))
article.append(addInsetCodeListing(["philip@raspberrypi:/dev/disk/by-partuuid $ lsblk", "NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT", "sda 8:0 0 4.5T 0 disk", "├─sda1 8:1 0 128M 0 part ", "└─sda2 8:2 0 4.5T 0 part /data", "sdb 8:16 0 232.9G 0 disk", "├─sdb1 8:17 0 256M 0 part /boot", "└─sdb2 8:18 0 232.6G 0 part", "sdc 8:32 1 7.5G 0 disk", "└─sdc1 8:33 1 7.5G 0 part /home/philip/usbstick", "mmcblk0 179:0 0 119.1G 0 disk", "├─mmcblk0p1 179:1 0 256M 0 part", "└─mmcblk0p2 179:2 0 118.8G 0 part /"]))
article.append(addParagraph("Note that the system boots from the MicroSD card (mmcblk0p1), sdb1 is a partition on a bootable external ssd."))
article.append(addParagraph("If I run blkid, I get the following results:"))
article.append(addInsetCodeListing(['philip@raspberrypi:~/usbstick $ blkid', '/dev/mmcblk0p1: LABEL_FATBOOT="boot" LABEL="boot" UUID="A764-6923" BLOCK_SIZE="512" TYPE="vfat" PARTUUID="64538962-01', '/dev/mmcblk0p2: LABEL="rootfs" UUID="2f4adfbf-32e5-44ec-9323-1d08cbe8c29e" BLOCK_SIZE="4096" TYPE="ext4" PARTUUID="64538962-02"', '/dev/sda2: LABEL="data" BLOCK_SIZE="512" UUID="F6426CBA426C80EB" TYPE="ntfs" PARTLABEL="Basic data partition" PARTUUID="059bc73a-1c97-4570-bc8b-7e7b6a5ba176"', '/dev/sdb1: LABEL_FATBOOT="boot" LABEL="boot" UUID="A764-6923" BLOCK_SIZE="512" TYPE="vfat" PARTUUID="64538962-01"', '/dev/sdb2: LABEL="rootfs" UUID="2f4adfbf-32e5-44ec-9323-1d08cbe8c29e" BLOCK_SIZE="4096" TYPE="ext4" PARTUUID="64538962-02"', '/dev/sdc1: LABEL="UUI" BLOCK_SIZE="512" UUID="96B02DA1B02D88B7" TYPE="ntfs" PARTUUID="0144129c-01"']))
article.append(addParagraph("This is essentially a summary of the information collected by the used space daemon so we have the id, the label, the partuuid the uuid so this can be a useful source of information as well."))
article.append(addParagraph("The sysfs (or sysfile) directory also contains some useful information. You might recall that this is one of the special file systems that can be mounted but is not a partition and its usual mount point is /sys. It contains useful information about hardware devices and drivers. For example, it has a subdirectory called block which contains information on each block device, including partitions, that the kernel can see."))
article.append(addParagraph("We can use the command, udevadm, to dig in to the information in the block directory. Let's look at an example of that."))
article.append(addInset("udevadm info -a -p /sys/block/sdb"))
article.append(addParagraph("This generates a lot of information (264 lines) regarding the device and if you want to examine it, you might consider redirecting the commands output to a file."))
article.append(addParagraph("A couple of final points I waat to make. Firstly, a partion uuid (partuuid) remains constant as long as a partition exists. If you reformat the drive, the partition will have a new uuid but the partuuid will not change. Again, it's important to understand the distinction between the uuid and partuuid and that before formatting, a partition already has a partuuid but it doesn't have a uuid until it has been formatted."))
article.append(addParagraph("Secondly, you may see a mount being referred to as efi which indicates that it is bootable."))
article.append(addParagraph("There is an interesting article on udev by Seth Kenlon on opensource.com called An introduction to Udev: The Linux subsystem for managing device events (13 November 2018)."))
article.append(addHeader("File System Types: ext4, Brtfs and XFS"))
article.append(addParagraph("There are lots of file systems and lots of differences between them such as whether they support journaling. A journal writes what you are going to do, that is, what changes you are going to make to a specific sector and it will also record the fact that the changes have been written (assuming everything runs correctly). If there is a problem such as a power failure, the journal can tell you what files were being written to and this may help you to locate corrupted files."))
article.append(addParagraph("Other differences include the file sizes that can be handled. For example, some can handle a lot of small files better and some can handle large file sizes better."))
article.append(addParagraph("Some also support copy-on-write which means that if you copy a file, the filesystem remaps it rather physically writing another copy so that can save on disk space."))
article.append(addParagraph("The most common file system on Linux is probably ext4 and this file system supports journaling. Its popularity is reflected in the fact that Linux has a man page for it."))
article.append(addParagraph("The Btrfs is fairly new but also quite popular. It supports copy-on-write and has lots of features."))
article.append(addParagraph("XFS is the default file system on RedHat 7. It supports journaling, including the jornaling of metadata so it records what you are writing to the file's metadata as well as to the file itself. It was originally deaigned to support large average file sizes so it may be a good xhoice for something like video editing."))
article.append(addParagraph("Linux also supposer FUSE (Filesystem in USErspace) file systems. As the name suggests, these are intended to allow non-privileged users to create file systems that don't affect the kernel code. File systems such as ext4 are part of the kernel code."))
article.append(addParagraph("They provide a lot of the features you would expect from a filesystem, such as a directory structure, but in reality it is acting as an interface to the underlying filesystem. An example of this type of system is SSHFS which allows you to SSH to a directory on a remote server as if it were a local directory so it can avoid the need to use tools such as NFS (Network File System)."))
article.append(addParagraph("Another good example of this which is not restricted entirely to Linux systems is Gmail which holds your data and looks like a file system."))
article.append(addParagraph("There is a lot of information on the comparative features of file systems on the Internet but so I'd like to mention some of these that look interesting."))
article.append(addSubHeader("Useful Links to More Info on Filesystems"))
article.append(addInsetCodeListing([" The Main Linux File Systems and Their Differences provides an overview of the major systems and has links to further reading.", " What Are the Best Linux Filesystems in 2021? was written by Odysseas Kourafalos and was last updated on January 2021. Also provides an overview of the major systems but is a bit less formal and looks like it provides a bit more detail.", " Comparison of file systems. This is a wikipedia page and it seems to be pertty comprehensive in terms of both the systems that are covered here and the detail which is more tabular than textual. It would be particularly useful if you need some information on an obscure filesystem. It is not restricted to Linux filesystems but also mentions systems used on Windows, Apple, Commodore and others and in most cases, there is a link to the wiki page on each individual system.", " Filesystem in Userspace is also a wikipedia page covering FUSE and it also includes a list of these systems (including SSFHS) and a link to the corresponding wiki page."]))
article.append(addHeader("Make File Systems"))
article.append(addParagraph("In this section, we are going to use a couple of very short scripts from the exercise files and these are format.sh and makefiles.sh. This involves working with a partition on the usb stick so here are a couple of links you may find useful in doing this."))
article.append(addInsetCodeListing([" Creating a disk partition in Linux from the fedoraproject.org.", " Resize Partitions in Linux from msp360.com."]))
article.append(addSubHeader("format.sh"))
article.append(addInsetCodeListing(["umount p10 # just in case", "if", "mkfs -t $1 /dev/sdc1", "then", "mount /dev/sdc10 ~/philip/p10", "else", "echo failed to format", "fi"]))
article.append(addParagraph("So this takes an argument which should be a filesystem type such as ext4, and it formats a partition - sda10 - with that fs and then mounts it to the directory p10."))
article.append(addSubHeader("makefiles.sh"))
article.append(addInsetCodeListing(["#!/bin/bash", "i=1", "while", "dd if=/dev/zero of=file$i bs=10K count=1 2>/dev/null 1>&2", "do", "i=$((i+1))", "done", "echo created $i files"]))
article.append(addParagraph("This file is effectively creating files using the dd command (dd stands for device dump) and each file is about 10k in size. It will continue to do this until the process fails at which time it will stop. We can then take a look at the partition and get some idea of how efficiently it is storing these files."))
article.append(addParagraph("Before we get started, I am now using a 32GB USB stick and it is device sda1 so it could be mounted with a command like"))
article.append(addInset("sudo mount /dev/sda1 /home/philip/usbstick"))
article.append("I have modified the format.sh program accordingly so it now looks like this.")
article.append(addInsetCodeListing(["umount /home/philip/usbstick # just in case", "if", "mkfs -t $1 /dev/sda1", "then", "mount /dev/sda1 /home/philip/usbstick", "else", "echo failed to format", "fi"]))
article.append(addParagraph("So we will go ahead and run the format.sh script, to make life a little easier I have modified all of the scripts in the exercise files folder to make them executable. That means I can run the command like this"))
article.append(addInset("format ext4"))
article.append(addParagraph("where ext4 is just the file type we want to use and if the command is successful, we shouldn't see any error message popping up. The output we do get is shown in this image."))
article.append(addImageWithCaption("./images/formatsh.png", "The output from running the format.sh command."))
article.append(addParagraph("So that seems to have worked pretty well. The next thing we want to do is navigate to the folder with the mounted drive and run makefiles.sh. Note that this file isn't in a folder where the shell would be looking for commands so I will provide its full path and I will also use time to check how long the process takes so the command is"))
article.append(addInset("sudo time /home/philip/Learning/linux_storage_systems/exercise_files/Ch01/makefiles.sh"))
article.append(addParagraph("Note that sudo is required here because the directory owner is root - I could also have just changed ownership to philip."))
article.append(addParagraph("As a matter of interest, when Kevin ran the command on his PC, it created 18809 files in around 10.5 seconds so it will be interesting to compare how the raspberry pi performs. The course is around 7 years old as of December 2023 so I would expect his PC to be reasonanly powerful."))
article.append(addParagraph("Actually, this took a little bit longer at around 4 and a quarter hours and generated a lot more files (1,880,470). I had forgotten that Kevin had set up a partition of around 200MB in order to run the scripts which is handy, because it gives me a change to review the partition creation process. Just to get this done quickly and easily, I am following a process detailed on a page on the phoenixap.com site on a page entitled How to Create Partitions in Linux and I'm only following this in so far as it helps me to create a partition and run the scripts again. I used sudo the command. I started by running the command"))
article.append(addInset("sudo parted -l"))
article.append(addInset("to get a list of current partitions and to make sure I was partitioning the USB stick and not the system disk! This confirms that the device label is sda. I actually created two partitions, one is about 15GB in size and I labelled it primary. The second is labelled secondary and is an extended disk with a size a little under 200MB and the partitions are sda1 and sda2 respectively. I used the command"))
article.append(addInset("sudo mkpart primary ext4 1MB 15000MB"))
article.append(addParagraph("to create the first partition and a similar command to create the second. Now that I have a suitable partition, I will amend the format.sh script to work with this smaller partiotion, sda2 and we'll run both scripts again. Note that before running makefiles again, I did change ownership of the usbstick folder to my own user account (philip) so I don't have to bother using sudo to run the command or if I want to delete the files created by the makefiles.sh script later. The output that is now generated by the command is shown below."))
article.append(addImageWithCaption("./images/makefilessh.png", "The output from running the makefiles.sh command."))
article.append(addParagraph("So that took a little bit longer, creating almost 18,000 files in about a minute and 25 seconds!"))
article.append(addParagraph("We can use the following command to get a better understanding of both the amount of space that is physically available and the amount of used space."))
article.append(addInset("df -h /home/philip/usbstick/"))
article.append(addParagraph("The output we get from this is shown below and remember, this is a partition with a size of around 200MB."))
article.append(addInsetList(["Filesystem 1K-blocks Used Available Use% Mounted on", "/dev/sdb3 192699 178361 2 100% /home/philip/usbstick"]))
article.append(addParagraph("This shows that the actual size is 189MB. We have used 175MB and there is around 2K of space remaining. There seems to be a difference between the overall size and the amount of space used of around 14MB and we are creating files of 10K until there is not enough space left for another file, the available space after completing the process should be less than 10K so the figure of 2K seems reasonable. Regardless, you can see that there is a good bit of overhead in that we all not able to use the entire 200 or so MB. We'll try formatting the partition again, but this time with the command"))
article.append(addInset("./format.sh \"btrfs -f\""))
article.append(addParagraph("Note that the -f option (force) is needed because we are formatting a partition that is already formatted with ext4. The filesystem and arguments are in quotes so that they are passed to the script as a single argument. This doesn't work on my Raspberry Pi which doesn't seem to have access to btrfs. I'm not going to go to the trouble of fixing that! Instead, I will just look at what happened with Kevin's partition."))
article.append(addParagraph("After formatting the partition with btrfs, Kevin ran the makefiles.sh script again. Just as a reminder, with ext4, this created 18,809 files in about 10.5 seconds. He then ran the command"))
article.append(addInset("df -k ."))
article.append(addParagraph("which shows the usage of space (in kilobytes) in the current directory and this showed that 190,139 kb had been used and 0 were available so usage was 100%."))
article.append(addParagraph("With btrfs, the results were very different. This time it created 15,374 files in about 9 seconds and the df -k command shows that 200,328 kb have been used with 4,476 kb free and usage is at 98%. If you run a command to create an emplty file such as"))
article.append(addInset("touch newfile"))
article.append(addParagraph("you will get a response saying that there is no space left on the device."))
article.append(addParagraph("As another experiment, the partition has been refomrmatted with ext4 again and Kevin ran the makelots.sh script which creates as many files as it can, but these are empty so we would expect that a lot more would be created."))
article.append(addParagraph("For Kevin, the process took around 22 seconds and created 51,190 files. The df -k command shows that 2% of the space has been used and around 170MB of available space. However, if we tried to created another new file, we get that no space left on device message again. When he ran the command"))
article.append(addInset("df -i ."))
article.append(addParagraph("this showed that inode usage is at 100%. Essentially, inodes is the number of files you can have on a particular file system which is 51,190 in this case. On my Raspberry Pi, this took around 67 seconds but it also produced 51,190 files as you can see below."))
article.append(addImageWithCaption("./images/makelotssh.png", "The output from running the makelots.sh command."))
article.append(addParagraph("Kevin reformatted his partition with btrfs and tried running makelots.sh again. This time, it took almost three and a half minutes and it created 388,112 files. Running df -k again shows that 98% of the space has been used and we have 4,476 kb of free space, the same as before."))
article.append(addParagraph("Running df -i shows no information about inodes. This is because unlike ext4 which created a fixed inodes table, btrfs just allocates space as it is needed and so the number of files you can have is limited only by the available space. In other words, you can continue creating new files as long as there is space available for the new file to be added."))
article.append(addParagraph("Kevin didn't say anything about why we still have that 4,476 kb of free space so I will assume that it is overhead from the btrfs file system - in other words, it has reserved that space for its own use which would explain why we saw the same amount free when running both makefiles.sh and makelots.sh. "))
article.append(addHeader("Mount File Systems Duting Boot"))
article.append(addParagraph("The /etc/fstab file contains a number of lines, each of which represents a device that will be mounted when the system starts up."))
article.append(addParagraph("This includes the partitions that are to be mounted and the mount points. The mount point is simply the directory where you want to meant the device."))
article.append(addParagraph("When you install Linux and you set up the partitions to be used for that installation, these are automatically included in the fstab file. This can be added to or edited post-installation."))
article.append(addParagraph("Each line in thefstab file consists of 6 fields."))
article.append(addInset("1 - What you want to mount."))
article.append(addParagraph("This can be a device like our USB stick or something else such as a remote file system or a blockfile."))
article.append(addInset("2 - The mount point."))
article.append(addParagraph("This would normally be an empty directory."))
article.append(addInset("3 - The file system type."))
article.append(addParagraph("We have already seen a couple of examples of this, ie ext4 and btrfs."))
article.append(addInset("4 - Mount options."))
article.append(addParagraph("Most of the time, the default options are fine but you can get some information on the more common options on the LINK1 page in the Debina docs."))
article.append(addInset("5 - Dump boolean."))
article.append(addParagraph("As far as I know, this option is not really used anymore. You can get some information about this in LINK2 on superuser.com."))
article.append(addInset("6 - File System Check Pass number."))
article.append(addParagraph("This determines the order in which disks are checked on start up. Since it is the most important, you would normally give the root partition a value of 1 with a 2 for everything else but if you have some drives or partitions you don't really care too much about, you could give them a higher value or a value of 0 (don't check)."))
article.append(addParagraph("The image below shows the /etc/fstab table from my web server as it is displayed by the cat command and in a slightly tidier version to make it easier to read."))
article.append(addImageWithCaption("./images/fstab.png", "The fstab file."))
article.append(addParagraph("This shows two partitions. The first is the boot partition which uses the vfat filesystem and the default mount options."))
article.append(addParagraph("The second is the partition on the USB stick I used in a previous exercise. It is currently formatted with ext4 and uses mostly the default options with the noatime option (do not update directory inode access times on the filesystem - this can help performance)."))
article.append(addParagraph(""))
main.append(menu("storage"))