Re: [git pull] more vfs bits

[David Howells <dhowells@redhat.com>]

Linus Torvalds <torvalds@linux-foundation.org> wrote:

> Also explain why that crap was done one file at a time?

Because it wasn't.  Here's the script for your perusal.  Al cherry-picked the
output, so you won't find everything the script produces in Al's pull request.

Breaking it down into one commit per fs makes it easier to review the
individual chunks.

David
---
#!/usr/bin/perl -w
use strict;

open(my $fd, "<$0") || die $0;
my @script = <$fd>;
close($fd);

my @file_system_types;
open(my $g, 'git grep -l "struct file_system_type.*=" |') ||
    die "Can't grep for filesystem type";
@file_system_types = <$g>;
close($g);

my @excludes = (
    "fs/attr.c",
    "fs/dcache.c",
    "fs/exportfs/expfs.c",
    "fs/file_table.c",
    "fs/notify/",
    "fs/locks.c",
    "fs/namei.c",
    "fs/namespace.c",
    "fs/open.c",
    "fs/utimes.c",
    "fs/xattr.c",
    "include/linux/",
    "Documentation/filesystems/vfs.txt",
    );

my @treat_as_fs = (
    "drivers/staging/lustre",
    "fs/kernfs",
    "fs/libfs.c",
    "fs/quota/dquot.c",
    "ipc",
    "kernel/relay.c",
    "kernel/trace",
    );

###############################################################################
#
# Find the filesystems and classify them according to whether they occupy a
# directory or a file in the source.
#
###############################################################################
my %fs_names = ();
my %fs_dirs = ();
my %fs_files = ();

# Miscellaneous convenience sets
my %fs_misc = (
#    "arch" => [],
#    "drivers" => [],
#    "fs" => [],
#    "security" => []
    );

my @fs_single = ();

fs_file: foreach my $file (@file_system_types) {
    chomp $file;
    foreach my $ex (@excludes, @treat_as_fs) {
	next fs_file if (substr($file, 0, length($ex)) eq $ex);
    }

    # Handle whole-directory filesystems
    if ($file =~ m!^fs/([a-z0-9]+)/.*[.]c!) {
	my $dir = substr($file, 0, rindex($file, "/"));
	my $name = $1;
	$fs_names{$name} = $dir;
	$fs_dirs{$dir} = [];
	next;
    }

    #next if ($file =~ m!^drivers/staging/lustre!);

    # Handle single-file filesystems
    $fs_files{$file} = [];
}

foreach my $path (@treat_as_fs) {
    if ($path =~ /[.][ch]/) {
	$fs_files{$path} = [];
    } else {
	my $name = substr($path, rindex($path, "/") + 1);
	$fs_names{$name} = $path;
	$fs_dirs{$path} = [];
    }
}

my @to_fs_inode = sort(keys(%fs_dirs));

###############################################################################
#
# Find all occurrences of files containing "->d_inode" and divide them amongst
# the various filesystems and non-filesystems.
#
###############################################################################
my @occurrences;
open($g, 'git grep -l "[-]>d_inode" |') ||
    die "Can't grep for ->d_inode";
@occurrences = <$g>;
close($g);

my %non_fs = ();

file: foreach my $file (@occurrences) {
    chomp $file;
    foreach my $ex (@excludes) {
	next file if (substr($file, 0, length($ex)) eq $ex);
    }

    foreach my $path (@to_fs_inode) {
	if (index($file, $path) == 0) {
	    #print $file, " found in ", $path, "\n";
	    push @{$fs_dirs{$path}}, $file;
	    next file;
	}
    }

    if (exists($fs_files{$file})) {
	foreach my $path (keys(%fs_misc)) {
	    if (index($file, $path) == 0) {
		push @{$fs_misc{$path}}, $file;
		delete $fs_files{$file};
		next file;
	    }
	}
    }

    if (exists($fs_files{$file})) {
	push @{$fs_files{$file}}, $file;
	next file;
    }

    if ($file =~ m!include/trace/events/([_a-zA-Z0-9]+)[.]h!) {
	my $fs = $1;
	if (exists($fs_names{$fs})) {
	    push @{$fs_dirs{$fs_names{$fs}}}, $file;
	    next;
	}
    }

    #print $file, " not found\n";
    $non_fs{$file} = [ $file ];
}

foreach my $path (sort(keys(%fs_files))) {
    push @fs_single, @{$fs_files{$path}};
}

###############################################################################
#
# Summarise how the filesystem file sets will be split up
#
###############################################################################
my $summarise = 0;
if ($summarise) {
    foreach my $path (sort(keys(%fs_dirs))) {
	print $path, ":\n";
	foreach my $file (@{$fs_dirs{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    foreach my $path (sort(keys(%fs_misc))) {
	print $path, "-single-fs:\n";
	foreach my $file (@{$fs_misc{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "single-fs:\n";
    foreach my $path (sort(keys(%fs_files))) {
	foreach my $file (@{$fs_files{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "non-filesystem:\n";
    foreach my $path (sort(keys(%non_fs))) {
	foreach my $file (@{$non_fs{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "\n";
}

###############################################################################
#
# Group the non-filesystems by directories with two or more files that need
# changing.
#
###############################################################################
my %non_groups = ();
my %non_dirs = ();

foreach my $file (keys(%non_fs)) {
    my $p = index($file, "/");
    my $q = index($file, "/", $p + 1);
    $p = $q if ($q != -1);
    my $dir = substr($file, 0, $p);
    $non_dirs{$dir} = 0 unless exists $non_dirs{$dir};
    $non_dirs{$dir}++;
    $non_groups{$dir} = [] unless exists $non_groups{$dir};
    push @{$non_groups{$dir}}, $file;
}

foreach my $dir (sort(keys(%non_dirs))) {
    #print $dir, " -> ", $non_dirs{$dir}, "\n";
    if ($non_dirs{$dir} == 1) {
	my $p = index($dir, "/");
	if ($p != -1) {
	    my $top = substr($dir, 0, $p);
	    $non_dirs{$top} = 0 unless exists $non_dirs{$top};
	    $non_dirs{$top}++;
	    $non_groups{$top} = [] unless exists $non_groups{$top};
	    push @{$non_groups{$top}}, @{$non_groups{$dir}};
	    delete $non_dirs{$dir};
	}
    }
}

#foreach my $dir (sort(keys(%non_dirs))) {
#    print "Non-filesystem ", $dir, ":\n";
#    foreach my $file (@{$non_groups{$dir}}) {
#	print "\t", $file, "\n";
#    }
#}

###############################################################################
#
# Set up the integration branch
#
###############################################################################
system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "reset", "--hard", "file-pin-devel") == 0 || die;

###############################################################################
#
# Fabricate commits for d_inode -> fs_inode() conversion
#
###############################################################################
system("git", "checkout", "file-pin-fs-experimental") == 0 || die;
die if `stg branch` ne "file-pin-fs-experimental\n";
system("git", "reset", "--hard", "file-pin") == 0 || die;

sub convert_to_fs_inode($$)
{
    my ($title, $files) = @_;

    unless (@{$files}) {
	print "Skipping $title with no files\n";
	return;
    }

    print "Process $title\n";

    my $dir = $files->[0];
    $dir =~ s![^/]+$!!;
    $dir =~ s!/$!!;
    $dir =~ s!/!_!g;

    foreach my $file (@{$files}) {
	open(my $fd, "<$file") || die $file;
	my @lines = <$fd>;
	close($fd);

	my @out = map {
	    s!ACCESS_ONCE[(](([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode[)]!fs_inode_once($1)!g;
	    s!(([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode!fs_inode($1)!g;
	} @lines;

	open($fd, ">$file") || die $file;
	print $fd @lines;
	close($fd) || die $file;
    }

    system("git", "add", @{$files}) == 0 || die;
    system("git", "commit", "-m",
	   "VFS: (Scripted) Convert ->d_inode to fs_inode() $title\n" .
	   "\n" .
	   'Signed-off-by: David Howells <dhowells@redhat.com>') == 0 || die;
}

foreach my $fs (sort(keys(%fs_dirs))) {
    convert_to_fs_inode("in $fs/", $fs_dirs{$fs});
}

foreach my $fs (sort(keys(%fs_misc))) {
    convert_to_fs_inode("in $fs/", $fs_misc{$fs});
}

#convert_to_fs_inode("miscellany", \@fs_single);
foreach my $file (sort(@fs_single)) {
    convert_to_fs_inode("in $file", [$file]);
}

# Merge the changes back into the integration branch, noting the script in the
# merge message.
my @msg = (
    "(Scripted) Merge in scripted filesystem ->d_inode to fs_inode() conversions\n",
    "\n",
    "Scripted merge in of scripted filesystem ->d_inode to fs_inode() conversions\n",
    "using the following perl script:\n",
    "\n",
    @script,
    "\n",
    'Signed-off-by: David Howells <dhowells@redhat.com>');

system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "merge", "--no-ff", "file-pin-fs-experimental", "-m", join("", @msg));

###############################################################################
#
# Fabricate an stg commit for d_inode -> dentry_inode() conversion
#
###############################################################################
system("git", "checkout", "file-pin-nonfs-experimental") == 0 || die;
die if `stg branch` ne "file-pin-nonfs-experimental\n";
system("git", "reset", "--hard", "file-pin") == 0 || die;

sub convert_to_dentry_inode($$)
{
    my ($title, $files) = @_;

    unless (@{$files}) {
	print "Skipping $title with no files\n";
	return;
    }

    print "Process $title\n";

    my $dir = $files->[0];
    $dir =~ s![^/]+$!!;
    $dir =~ s!/$!!;
    $dir =~ s!/!_!g;

    foreach my $file (@{$files}) {
	open(my $fd, "<$file") || die $file;
	my @lines = <$fd>;
	close($fd);

	my @out = map {
	    s!ACCESS_ONCE[(](([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode[)]!dentry_inode_once($1)!g;
	    s!(([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode!dentry_inode($1)!g;
	} @lines;

	open($fd, ">$file") || die $file;
	print $fd @lines;
	close($fd) || die $file;
    }

    system("git", "add", @{$files}) == 0 || die;
    system("git", "commit", "-m",
	   "VFS: (Scripted) Convert ->d_inode to dentry_inode() $title\n" .
	   "\n" .
	   'Signed-off-by: David Howells <dhowells@redhat.com>') == 0 || die;
}

foreach my $dir (sort(keys(%non_dirs))) {
    convert_to_dentry_inode("in $dir", $non_groups{$dir});
}

# Merge the changes back into the integration branch, noting the script in the
# merge message.
@msg = (
    "(Scripted) Merge in scripted non-filesystem ->d_inode to dentry_inode() conversions\n",
    "\n",
    "Scripted merge in of scripted non-filesystem ->d_inode to dentry_inode() conversions\n",
    "using the following perl script:\n",
    "\n",
    @script,
    "\n",
    'Signed-off-by: David Howells <dhowells@redhat.com>');

system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "merge", "--no-ff", "file-pin-nonfs-experimental", "-m", join("", @msg));