#!/usr/bin/perl -w
use strict;
$|=1;

use EVEcommon;
use Data::Dumper;
use Carp;

my $datadir = @ARGV ? shift @ARGV : "cache";
$datadir =~ s,(/*$),/,;
die "bad datadir $datadir" unless -d $datadir;

# prereq attributes
my @prereqs = (
	[1,182,277,],
	[2,183,278,],
	[3,184,279,],
	[4,1285,1286,],
	[5,1289,1287,],
	[6,1290,1288,],);

&load_units();
&load_graphics();
&load_metagroups();
&load_marketgroups();
&load_variations();
&load_attributes();
&load_categories();
&load_groups();
&load_types();
&load_typeattributes();
&load_typeeffects();
&load_certs();
&load_certrels();
&load_certrecs();
&load_certcats();
&load_certclasses();


### CERTS

open(CF, ">", "eve-certificates.xml") or die $!;
print CF '<?xml version="1.0"  encoding="UTF-8"?>',"\n",'<certificates>',"\n";

my @ccs = &certcats();
for my $ccat (sort {$a->{categoryName} cmp $b->{categoryName}} @ccs) {
	printf CF '  <certificateCategory name="%s" id="%s" descr="%s">%s',
		$ccat->{categoryName}, 
		$ccat->{categoryID}, 
		&render_descr($ccat->{description}),
		"\n";

	my %ccc = ();
	my @certs = &certs_by_catid($ccat->{categoryID});

	for my $c (@certs) {
		my $cid = $c->{certificateID};
		my $grade = $c->{grade};
		my $class = $c->{classID};
		die "cert without class" unless $class;
		$ccc{$class} ||= {};

#printf "DEB: ccc{$class}{$grade}\n";
		die "dupe ccc{$class}{$grade}" if $ccc{$class}{$grade};

		$ccc{$class}{$grade} = $c;
	}


	for my $classid (sort {$a <=> $b} keys %ccc) {
		my $cclass = &certclass_by_id($classid);
		die "no class for $classid" unless $cclass;
		die "classid mismatch" unless $cclass->{classID} == $classid;

		printf CF '    <certificateClass name="%s" id="%s" descr="%s">%s',
			$cclass->{className}, 
			$cclass->{classID}, 
			&render_descr($cclass->{description}),
			"\n";


		for my $grade (sort {$a <=> $b} keys %{$ccc{$classid}}) {
			my $cert = $ccc{$classid}{$grade};
			die "missing ccc{$classid}{$grade}" unless $cert;

			printf CF '      <certificate grade="%s" id="%s" descr="%s">%s',
				&level_to_grade($grade),
				$cert->{certificateID}, 
				&render_descr($cert->{description}),
				"\n";

			my @rec = &certrecs_by_certid($cert->{certificateID});
			for my $r (@rec) {
				printf CF '        <recommendation id="%s" level="%s" ship="%s"/>%s',
					$r->{recommendationID},
					$r->{recommendationLevel},
					&type_by_id($r->{shipTypeID})->{typeName},
					"\n";
			}

			my @rel = &certrels_by_certid($cert->{certificateID});
			for my $r (@rel) {
				my $rid = $r->{relationshipID};
				my $ptid = ($r->{parentTypeID} > 0) ? $r->{parentTypeID} : 0;
				my $pid = ($r->{parentID} > 0) ? $r->{parentID} : 0;
				die "both type and id for rid $rid" if $ptid && $pid;
				my ($type, $name, $level,);
				if ($pid) {
					$type = "cert";
					my $cert = &cert_by_id($pid);
					$name = &certclass_by_id($cert->{classID})->{className};
					$level = &level_to_grade($cert->{grade});
				} elsif ($ptid) {
					$type = "type";
					$name = &type_by_id($ptid)->{typeName};
					$level = $r->{parentLevel};
				} else {
					die "never here";
				}
				printf CF '        <requires id="%s" type="%s" name="%s" level="%s"/>%s',
					$r->{relationshipID},
					$type, $name, $level,
					"\n";
			}
			printf CF "      </certificate>\n";
		}
		printf CF "    </certificateClass>\n";
	}
	printf CF "  </certificateCategory>\n";
}
print CF "</certificates>\n";
close CF or die $!;


#### START ITEMS

my $tree = {
	group	=> { marketGroupID => 0, marketGroupName => 'Ship Items', },
	subs	=> [],
	items	=> [],
	};
my %groups = ( NULL => $tree );
#|             9 |          NULL | Ship Equipment         |
#|            11 |          NULL | Ammunition & Charges   |
#|            24 |          NULL | Implants & Boosters    |
#|           157 |          NULL | Drones                 |
#|           477 |          NULL | Starbase Structures    |
#|           955 |          NULL | Ship Modifications     |
#for (9, 11, 24, 157, 477, 955,) {
#	my $mg = &marketgroup_by_id($_);
for (	"Ship Equipment",
	"Ammunition & Charges",
	"Implants & Boosters",
	"Drones",
	"Starbase Structures",
	"Ship Modifications",
	) {
	my $mg = &marketgroup_by_name($_);
	my $mgid = $mg->{marketGroupID};
	die "no mgid for $_" unless $mgid;
	die "dupe $_" if $groups{$mgid};
	my $grp = { group => $mg, subs => [], items => [], };
	push @{$tree->{subs}}, $grp;
	$groups{$mgid} = $grp;
}

my $ac = 1;
while ($ac) {
	$ac = 0;
	for (&marketgroup_ids()) {
		next if $groups{$_};
		my $mg = &marketgroup_by_id($_);
		my $pid = $mg->{parentGroupID};
		next unless $pid && $groups{$pid};
		my $grp = { group => $mg, subs => [], items => [], };
		push @{$groups{$pid}->{subs}}, $grp;
		$groups{$_} = $grp;
		$ac++;
	}
	printf "TREELOOP: %i added\n", $ac;
}

my %chargegroups =();
for (&groups_by_category("Charge")) {
	my $gid = $_->{groupID};
	die "dupe gid $gid" if $chargegroups{$gid};
#	my $g = &group_by_id($_);
#next unless $g->{published};
	$chargegroups{$gid} = $_;
}

my %types = ();
for (&type_ids()) {
	die "dupe tid $_" if $types{$_};
	my $t = &type_by_id($_);
	next unless $t->{published};
	$types{$_} = $t;

	my $g = $t->{groupID};
	if ($g && $chargegroups{$g} && (!defined $t->{marketGroupID} || !$t->{marketGroupID})) {
		printf "CHARGE-NO-MARKETGROUP: %i '%s'\n",
			$_, $t->{typeName};
	}
}
printf "LOADED %i types\n", scalar keys %types;

my %typegroup = ();
# PASS 1 - assign by marketgroupID
$ac = 0;
my $dc = 0;
for (keys %types) {
	my $t = $types{$_};
	my $mgid = $t->{marketGroupID};
	next unless $mgid;
	delete $types{$_};
	$dc++;
	next unless $groups{$mgid};
	die "wtf BP $_" if $t->{typeName} =~ /Blueprint/;
	push @{$groups{$mgid}->{items}}, $t;
	die "unexpeced tg $_" if $typegroup{$_};
	$typegroup{$_} = $groups{$mgid};
	$ac++;
}
printf "PASS1 (marketgroups) %i added, %i deleted, %i left\n", $ac, $dc-$ac, scalar keys %types;

# PASS 2 - assign by variation
($ac, $dc,) = (0,0,);
for (keys %typegroup) {
	my $tid = $_;
	my $grp = $typegroup{$_};
	die "unexpeced type $_" if $types{$_};
	for (&variations_by_id($tid)) {
		my $vid = $_;
		next if $typegroup{$vid};
		unless ($types{$vid}) {
			$types{$vid} = &type_by_id($vid);
			die "missing vid $vid" unless $types{$vid};
#			warn "summoned variation $vid for $tid";
			$dc++;
		}
		my $t = delete $types{$vid};
		die "wtf BP $vid for $tid" if $t->{typeName} =~ /Blueprint/;
		push @{$grp->{items}}, $t;
		$typegroup{$vid} = $grp;
		$ac++;
	}

}
printf "PASS2 (variations) %i added, %i summoned, %i left\n", $ac, $dc, scalar keys %types;

# PASS 3 - assign by having a slot id
($ac, $dc,) = (0,0,);
for (keys %types) {
	my $tid = $_;
	my $ta = &typeattributes($tid,1);
	next unless $ta->{331};
	die "unexpeced tg $tid" if $typegroup{$tid};
	die "high slot ".$ta->{331} if $ta->{331} > 10;
	my $mgid = 617 + $ta->{331};
	die "bad mgid $mgid" unless $groups{$mgid};
	my $t = delete $types{$tid};
	die "wtf BP $tid" if $t->{typeName} =~ /Blueprint/;
	my $grp = $groups{$mgid};
	push @{$grp->{items}}, $t;
	$typegroup{$tid} = $grp;
	$ac++;
}
printf "PASS3 (implantslot) %i added, %i left\n", $ac, scalar keys %types;

# PASS 4 - assign by being in the same group as a known item
my %structgroups = ();
my @ss = (
	$groups{11},  # Charges
	$groups{477}, # Structures
	);
while (@ss) {
	my $grp = shift @ss;
	my $mgid = $grp->{group}->{marketGroupID};
	for my $t (@{$grp->{items}}) {
		my $tid = $t->{typeID};
		my $gid = $t->{groupID};
		if ($structgroups{$gid}) {
			warn "redefined gid $gid from mgid ".
				$structgroups{$gid}." to $mgid" 
				unless $structgroups{$gid} == $mgid;
		}
		$structgroups{$gid} = $mgid;
#		unless (&variations_by_id($tid)) {
#			printf "NOVAR: %5i '%s'\n", $tid, $t->{typeName};
#		}
	}
	push @ss, @{$grp->{subs}};
}
#printf "have %i struct groups\n", scalar keys %structgroups;

($ac, $dc,) = (0,0,);
for (keys %types) {
	my $tid = $_;
	my $t = $types{$tid};
	my $gid = $t->{groupID};
	die "missing gid for $tid" unless $gid;
	next unless $structgroups{$gid};
	die "wtf BP $tid" if $t->{typeName} =~ /Blueprint/;
	my $mgid = $structgroups{$gid};
	die "bad mgid $mgid" unless $groups{$mgid};
	delete $types{$tid};
	my $grp = $groups{$mgid};
	push @{$grp->{items}}, $t;
	$typegroup{$tid} = $grp;
	$ac++;
	printf "PASS4 added: %i '%s'\n",
		$t->{typeID}, $t->{typeName};
#	unless (&variations_by_id($tid)) {
#		printf "NOVAR: %5i '%s'\n", $tid, $t->{typeName};
#	}
}
printf "PASS4 (market group groups) %i added, %i left\n", $ac, scalar keys %types;


printf "ASSIGNED %i ITEMS\n", scalar keys %typegroup;
#for (keys %typegroup) {
#	my $tid = $_;
#	my $t = &type_by_id($tid);
#	printf "TYPE: %s\n", $t->{typeName};
#}

open(SF, ">", "eve-items2.xml") or die $!;
print SF '<?xml version="1.0"?>',"\n";
&render_group($tree);
close SF or die $!;

sub render_group ($$) {
	my ($tree, $depth,) = @_;
	$depth ||= 0;
	if ($depth < 2) {
		printf "%s%04i %s (%i items)\n", 
			"    "x$depth, 
			$tree->{group}->{marketGroupID},
			$tree->{group}->{marketGroupName},
			scalar @{$tree->{items}};
	}

	printf SF "%s<ItemCategory>\n", "    "x$depth;
	printf SF "%s  <Name>%s</Name>\n", "    "x$depth,
			&render_descr($tree->{group}->{marketGroupName});

	if (@{$tree->{subs}}) {
		printf SF "%s  <Subcategories>\n", "    "x$depth;
		for (sort {$a->{group}->{marketGroupName} cmp $b->{group}->{marketGroupName}} @{$tree->{subs}}) {
			&render_group($_, $depth+1);
		}
		printf SF "%s  </Subcategories>\n", "    "x$depth;
	}

	if (@{$tree->{items}}) {
		printf SF "%s  <Items>\n", "    "x$depth;
		for (sort {$a->{typeName} cmp $b->{typeName}} @{$tree->{items}}) {
			&render_item($_, $depth+1);
		}
		printf SF "%s  </Items>\n", "    "x$depth;
	}

	printf SF "%s</ItemCategory>\n", "    "x$depth;
}

sub render_item ($$) {
	my ($type, $depth,) = @_;
	$depth ||= 0;
	my $tid = $type->{typeID};
	my $ta = &typeattributes($tid);
	my %a = %$ta;

#	printf SF "%s>>>> %s\n", "    "x$depth, $type->{typeName};
#	return;

	my $mg = &metagroup_by_typeid($tid);
	my $dmg = "Tech I";
	if ($mg) {
		if ($mg->{metaGroupID} == 1) {
			$dmg = 'Named';
		} else {
			$dmg = $mg->{metaGroupName};
		}
	} elsif (defined $ta->{422}) {
		my $tl = $ta->{422};
		$mg = {
			1 => "Tech I",
			2 => "Tech II",
			3 => "Tech III",
			}->{$tl};
		if ($mg) {
			$dmg = $mg;
		} else {
			warn "bad tl:$tl for tid:$tid";
		}
	}
#	die "cant decide on metaGroup for $tid" unless $dmg;

	my $icon = &get_icon_by_type($type);

	printf SF '%s<Item Id="%s" Name="%s" Description="%s" Icon="%s" Metagroup="%s" >%s',
			"    "x$depth,
			$type->{typeID},
			&render_descr($type->{typeName}),
			&render_descr($type->{description}),
			$icon,
			$dmg,
			"\n";

	printf SF "%s  <RequiredSkills>\n", "    "x$depth;
	for (@prereqs) {
		my ($ai, $as, $al,) = @$_;
		my $s = defined $a{$as} ? delete $a{$as} : 0;
		my $l = defined $a{$al} ? delete $a{$al} : 0;
		if ($tid == 30651) {
			printf "DEB(%i): s(%i => %i), l(%i => %i)\n",
				$tid, $as, $s, $al, $l;
		}
		next unless $s && $l;
		printf SF '%s    <skill Name="%s" Level="%s"/>%s',
			"    "x$depth, 
			&type_by_id($s)->{typeName},
			$l, "\n";
	}
	printf SF "%s  </RequiredSkills>\n", "    "x$depth;


	printf SF "%s  <Properties>\n", "    "x$depth;
	printf SF '%s    <prop Name="Base price" Value="%s ISK" />%s', "    "x$depth,
			&render_attrval($type->{basePrice}, {}), "\n";
	printf SF '%s    <prop Name="Mass" Value="%s kg" />%s', "    "x$depth,
			&render_attrval($type->{mass}, {}), "\n";
	printf SF '%s    <prop Name="Volume" Value="%s m3" />%s', "    "x$depth,
			&render_attrval($type->{volume}, {}), "\n";
	printf SF '%s    <prop Name="Capacity" Value="%s m3" />%s', "    "x$depth,
			&render_attrval($type->{capacity}, {}), "\n";


	my $eff = &typeeffects($tid, 1);
	if (defined $eff->{11}) {
		die "multislot" if defined $eff->{12} || defined $eff->{13};
		printf SF '%s    <prop Name="Slot type" Value="Low slot" />%s', "    "x$depth, "\n";
	}
	if (defined $eff->{12}) {
		die "multislot" if defined $eff->{11} || defined $eff->{13};
		printf SF '%s    <prop Name="Slot type" Value="High slot" />%s', "    "x$depth, "\n";
	}
	if (defined $eff->{13}) {
		die "multislot" if defined $eff->{11} || defined $eff->{12};
		printf SF '%s    <prop Name="Slot type" Value="Mid slot" />%s', "    "x$depth, "\n";
	}


	for my $aid (sort {&attribute_by_id($a)->{attributeName} cmp &attribute_by_id($b)->{attributeName}} keys %a) {
		my $v = delete $a{$aid};
		if ($aid == 107) {
			warn "BR69169: pre-QR explosion radius";
			next;
		}
		my $a = &attribute_by_id($aid);
		my $n = $a->{displayName} || $a->{attributeName};
#		next if !$v && $n =~ /^(CPU Load|Damage|Powerload)$/;
		my $dv = &render_attrval($v, $a);
#			next unless $s && $l;
		printf SF '%s    <prop Name="%s" Value="%s" />%s', "    "x$depth, $n, $dv, "\n";
	}
	printf SF "%s  </Properties>\n", "    "x$depth;

	for my $aid (sort {$a <=> $b} keys %a) {
		my $attr = &attribute_by_id($aid);
		printf "TypeAttributes(%i): %s %s\n", $aid, 
				$attr->{attributeName},
				$ta->{$aid};
	}

	printf SF "%s</Item>\n", "    "x$depth;
}


#print Dumper($tree);




#exit 23;

###

open(SF, ">", "eve-ships2.xml") or die $!;
print SF '<?xml version="1.0"?>',"\n",
	'<ArrayOfShip xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">',"\n";

my %ships = ();
my @shg = &groups_by_category("Ship");
for my $group (@shg) {
#	printf "Group(%i): %s\n", $group->{groupID}, $group->{groupName};
	my @st = &types_by_group($group->{groupID});
	for my $type (@st) {
#		printf "Type(%i): %s\n", $type->{typeID}, $type->{typeName};
		next unless $type->{published};
		my $ta = &typeattributes($type->{typeID});
		die "dupe type: ".$type->{typeName} if $ships{$type->{typeName}};
		$ships{$type->{typeName}} = [$group, $type, $ta,];
	}
}

for (sort keys %ships) {
	my ($group, $type, $ta,) = @{$ships{$_}};
#		printf "Type(%i): %s\n", $type->{typeID}, $type->{typeName};
		my %a = %$ta;

		my $race = "UNKNOWN";
		if (!defined $type->{marketGroupID} || !$type->{marketGroupID}) {
			$race = 'Faction';
		} else {
			$race = &marketgroup_by_id($type->{marketGroupID})->{marketGroupName};
		}

		# checky-checky
		&get_icon_by_type($type);

		printf SF ' <Ship Id="%s" Name="%s" Description="%s" Race="%s" Type="%s">%s',
			$type->{typeID},
			$type->{typeName},
			&render_descr($type->{description}),
			$race,
			$group->{groupName},
			"\n";

		print SF "  <RequiredSkills>\n";
		for (@prereqs) {
			my ($ai, $as, $al,) = @$_;
			my $s = defined $a{$as} ? delete $a{$as} : 0;
			my $l = defined $a{$al} ? delete $a{$al} : 0;
			next unless $s && $l;
			printf SF '   <skill Name="%s" Level="%s"/>%s',
				&type_by_id($s)->{typeName},
				$l, "\n";
		}
		print SF "  </RequiredSkills>\n";


		print SF "  <Properties>\n";
		printf SF '   <prop Name="Base price" Value="%s ISK" />%s',
			&render_attrval($type->{basePrice}, {}), "\n";
		printf SF '   <prop Name="Mass" Value="%s kg" />%s',
			&render_attrval($type->{mass}, {}), "\n";
		printf SF '   <prop Name="Volume" Value="%s m3" />%s',
			&render_attrval($type->{volume}, {}), "\n";
		printf SF '   <prop Name="Capacity" Value="%s m3" />%s',
			&render_attrval($type->{capacity}, {}), "\n";

		for my $aid (sort {&attribute_by_id($a)->{attributeName} cmp &attribute_by_id($b)->{attributeName}} keys %a) {
			my $v = delete $a{$aid};
			my $a = &attribute_by_id($aid);
			my $n = $a->{displayName} || $a->{attributeName};
			next if !$v && $n =~ /^(CPU Load|Damage|Powerload)$/;
			my $dv = &render_attrval($v, $a);
#			next unless $s && $l;
			printf SF '   <prop Name="%s" Value="%s" />%s', $n, $dv, "\n";
		}
		print SF "  </Properties>\n";

		for my $aid (sort {$a <=> $b} keys %a) {
			my $attr = &attribute_by_id($aid);
			printf "TypeAttributes(%i): %s %s\n", $aid, 
					$attr->{attributeName},
					$ta->{$aid};
		}

		print SF " </Ship>\n";
#		last;
}
print SF "</ArrayOfShip>\n";
close SF or die $!;


#exit 42;

###

open(SF, ">", "eve-implants2.xml") or die $!;
print SF '<?xml version="1.0"?>',"\n",
	'<ArrayOfSlot xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema">',"\n";

my %implants = ();
my @ig = &groups_by_category("Implant");
for my $group (@ig) {
	printf "Group(%i): %s\n", $group->{groupID}, $group->{groupName};
	my @st = &types_by_group($group->{groupID});
	for my $type (@st) {
#		printf "Type(%i): %s\n", $type->{typeID}, $type->{typeName};
		my $ta = &typeattributes($type->{typeID});
#		next unless scalar %$ta;

		if (defined $ta->{331}) {
			my $slot = $ta->{331};
			$implants{$slot} ||= [];
			push @{$implants{$slot}}, [$type, $ta,];
			next;
		};

		next if $group->{groupName} eq "Booster";
		printf "NoSlot(typeID:%i): %s\n", $type->{typeID}, $type->{typeName};

#		for my $aid (sort {$a <=> $b} keys %$ta) {
#			my $attr = &attribute_by_id($aid);
#			printf "TypeAttributes(%i): %s %s\n", $aid, 
#					$attr->{attributeName},
#					$ta->{$aid};
#		}

	}
#	exit 5;
}

my @slotmap = (
	[ 1, "Perception",	178,	],
	[ 2, "Memory",		177,	],
	[ 3, "Willpower",	179,	],
	[ 4, "Intelligence",	176,	],
	[ 5, "Charisma",	175,	],
	[ 6, "Misc",		-1,	],
	[ 7, "Misc",		-1,	],
	[ 8, "Misc",		-1,	],
	[ 9, "Misc",		-1,	],
	[10, "Misc",		-1,	],
	);
for (@slotmap) {
	my ($slot, $attr, $pba,) = @$_;
	printf "SLOT(%s): %i plants\n", $slot, scalar @{$implants{$slot}};
	printf SF ' <Slot Number="%s" a="%s">%s', $slot, $attr, "\n";

	for (sort {$a->[0]->{typeName} cmp $b->[0]->{typeName}} @{$implants{$slot}}) {
		my ($type, $ta, ) = @$_;
#		printf "Type(%i): %s\n", $type->{typeID}, $type->{typeName};
		my %a = %$ta;

		my $bonus = $a{$pba} ? delete $a{$pba} : 0;
		die "no pba" if $pba > 0 && !$bonus;
		for (175..179) {
			delete $a{$_} if defined $a{$_} && !$a{$_};
		}

		my $sl = delete $a{331};
		die "wrong slot $sl != $slot" unless $sl == $slot;

		my $tl = defined $a{422} ? delete $a{422} : 0;

		printf SF '  <Implant Tech2="%s" Bonus="%s" ID="%s" Icon="icon%s.png" Name="%s" Description="%s">%s',
			$tl == 2 ? "true" : "false",
			$bonus, $type->{typeID},
			&graphic_by_id($type->{graphicID})->{icon},
			$type->{typeName},
			&render_descr($type->{description}),
			"\n";

		print SF "   <RequiredSkills>\n";
		for (@prereqs) {
			my ($ai, $as, $al,) = @$_;
			my $s = defined $a{$as} ? delete $a{$as} : 0;
			my $l = defined $a{$al} ? delete $a{$al} : 0;
			next unless $s && $l;
			printf SF '    <skill Name="%s" Level="%s"/>%s',
				&type_by_id($s)->{typeName},
				$l, "\n";
		}
		print SF "   </RequiredSkills>\n";

		if (scalar %a) {
			print SF "   <Properties>\n";
			for my $a (sort {$a <=> $b} keys %a) {
				my $v = delete $a{$a};
#				next unless $s && $l;
				printf SF '    <prop Name="%s" Value="%s"/>%s',
					&attribute_by_id($a)->{attributeName},
					$v, "\n";
			}
			print SF "   </Properties>\n";
		}

		for my $aid (sort {$a <=> $b} keys %a) {
			my $attr = &attribute_by_id($aid);
			printf "TypeAttributes(%i): %s %s\n", $aid, 
					$attr->{attributeName},
					$ta->{$aid};
		}

		print SF "  </Implant>\n";
#		last;
	}
	print SF " </Slot>\n";
}
print SF "</ArrayOfSlot>\n";
close SF or die $!;


#exit 42;

###

open(SF, ">", "eve-skills2.xml") or die $!;
print SF '<?xml version="1.0"  encoding="UTF-8"?>',"\n",'<skills>',"\n";

my %rbp = ();
my @sg = &groups_by_category("Skill");
for my $group (sort {$a->{groupName} cmp $b->{groupName}} @sg) {
	next if $group->{groupName} eq "Fake Skills";
	printf "SkillGroup(%i): %s\n", $group->{groupID}, $group->{groupName};
	printf SF '  <c n="%s" g="%s">%s', $group->{groupName}, $group->{groupID}, "\n";
	my @st = &types_by_group($group->{groupID});
	for my $type (sort {$a->{typeName} cmp $b->{typeName}} @st) {
#		printf "SkillType(%i): %s\n", $type->{typeID}, $type->{typeName};
		my $ta = &typeattributes($type->{typeID});
		next unless scalar %$ta;

#		for my $aid (sort {$a <=> $b} keys %$ta) {
#			my $attr = &attribute_by_id($aid);
#			printf "TypeAttributes(%i): %s %s\n", $aid, 
#					$attr->{attributeName},
#					$ta->{$aid};
#		}

		my $descr = &render_descr($type->{description});

		die "missing a180" unless $ta->{180};
		die "missing a181" unless $ta->{181};
		die "missing rank" unless $ta->{275};
		die "missing price" unless defined $type->{basePrice};

		if ($type->{typeName} =~ /^((?:Amarr|Caldari|Gallente|Minmatar)\w*) (.+)$/) {
			my $bp = $type->{basePrice};
			die "redefining rbp for ".$type->{typeName} if $rbp{$2}{$1};
			$rbp{$2}{$1} = $bp;
		} elsif ($type->{typeName} =~ /(Amarr|Caldari|Gallente|Minmatar)/i) {
			die "racial mismatch: ".$type->{typeName};
		}

		my $have_prereq = 0;
		for (@prereqs) {
			my ($ai, $as, $al,) = @$_;
			die "sl$ai without s$ai" if !$ta->{$as} && $ta->{$al};
			$have_prereq++ if $ta->{$al};
			if ($ta->{$as} && !$ta->{$al}) {
				my $pl = defined $ta->{$al} ? $ta->{$al} : "undef";
				my $pn = &type_by_id($ta->{$as})->{typeName};
				my $n = $type->{typeName};
				warn "false($pl) prereq($ai) '$pn' for '$n'";
			}
		}

		my $rank = int($ta->{275}+0.5);
		warn "rank for ".$type->{typeName}." changed from ".$ta->{275}." to $rank" 
			unless $ta->{275} == $rank;

		printf SF '    <s %sn="%s" i="%s" d="%s" a1="%s" a2="%s" r="%s" c="%s" canTrainOnTrial="%s"',
			$type->{published} ? "" : 'p="false" ',
			$type->{typeName}, $type->{typeID},
			$descr,
			&attribute_by_id($ta->{180})->{attributeName},
			&attribute_by_id($ta->{181})->{attributeName},
			$rank,
			$type->{basePrice},
			$ta->{1047} ? 'false' : 'true';

		unless ($have_prereq) {
			print SF "/>\n";
			next;
		}

		print SF ">\n";

		for (@prereqs) {
			my ($ai, $as, $al,) = @$_;

			if ($ta->{$al}) {
				printf SF '      <p n="%s" l="%s" />%s', 
					&type_by_id($ta->{$as})->{typeName},
					$ta->{$al},
					"\n";
			}
		}
		print SF "    </s>\n";
	}
	print SF "  </c>\n";
#	exit 5;
}
print SF "</skills>\n";
close SF or die $!;


for my $s (sort keys %rbp) {
	my $bp = undef;
	for my $r (sort keys %{$rbp{$s}}) {
		my $rbp = $rbp{$s}{$r};
		$bp = $rbp unless defined $bp;
		warn "BP mismatch for '$r $s'" unless $rbp == $bp;
	}
}



exit 0;

##

sub get_icon_by_type ($) {
	my $type = shift;
	my $tid = $type->{typeID};
	die "no typeID in $type" unless $tid;

	my $iconfile = $tid;
	my $icon = sprintf "/icons/64_64/%s.png", $iconfile;
	if (-e ".".$iconfile) {
		my $big = sprintf "/icons/256_256/%s.png", $iconfile;
		warn "MISSING: BIG $big" unless -e ".".$big;
	} elsif ($type->{graphicID}) {
		my $graphic = &graphic_by_id($type->{graphicID});
		my $ic = $graphic->{icon};
		if ($ic) {
			$iconfile = "icon".$ic;
		}
		$icon = sprintf "/icons/64_64/%s.png", $iconfile;
		if ($type->{published}) {
			warn "MISSING: ICON $icon" unless -e ".".$icon;
		}
	} else {
		warn "MISSING: ALL $tid";
	}
	return $iconfile;
}


sub render_attrval ($$) {
	my ($v, $a,) = @_;
	my $form = "%s %s";
	my $u = $a->{unitID} ? &unit_by_id($a->{unitID}) : {};
	my $du = $u->{displayName} || $u->{unitName} || "";
	my $dv = $v;

	# clobber milliseconds
	if ((($a->{unitID}||0) == 101) ||
	    ($du eq "ms" && $dv =~ /000$/)) {
		$du = "sec";
		$dv /= 1000;
	}

	# BR 71720
	if ($du eq "sec" &&
            $dv > 5000) {
		my $n = "attributeID:".$a->{attributeID}." ".
			($a->{displayName} || $a->{attributeName});
		if ($a->{attributeName} =~ /^(boosterDuration|missileLaunchDuration|warpScrambleDuration|cloakingTargetingDelay)$/) {
			warn "BR71720: fixing $dv sec '$n'";
			$dv /= 1000;
		} else {
#			warn "IGNORING BR71720 CANDIDATE: $dv sec '$n'";
		}
	}

	my $d = 0;
	if (($u->{unitName}||"") eq "Inverse Absolute Percent") {
		$dv = (1.0-$dv)*-100;
	} elsif (($u->{unitName}||"") eq "Modifier Percent") {
		$dv = ($dv-1.0)*100;
	} elsif (($u->{unitName}||"") eq "Absolute Percent") {
		$dv = $dv*100;
#$d=1;
	} elsif (($u->{unitName}||"") eq "Inversed Modifier Percent") {
		$dv = (1.0-$dv)*100;
	} elsif (($u->{unitName}||"") eq "typeID") {
		my $g = &type_by_id($v);
		$dv = &render_descr($g->{typeName});
		$du = "";
	} elsif (($u->{unitName}||"") eq "groupID") {
		my $g = &group_by_id($v);
		$dv = &render_descr($g->{groupName});
		$du = "";
	} elsif (($u->{unitName}||"") eq "Sizeclass") {
		$dv = { 
			1 => 'Small',
			2 => 'Medium',
			3 => 'Large',
			4 => 'Extra Large',
			}->{$v};
		die "bad SizeClass $v" unless $dv;
		$du = "";
	}

	if ($dv !~ /^0\.\d+$/ &&
	    $dv =~ /\.(\d+)$/ &&
	    int($dv) != $dv) {
	    	if (length($1) > 5) {
			$dv = sprintf "%.3f", $dv;
		} else {
			$dv = sprintf "%f", $dv;
		}
		$dv =~ s/0+$//;
		$dv =~ s/\.$//;
      	} else {
		if ($du eq 'sec' && $dv =~ /\./) {
			warn "DEB: secnomatch '$dv'";
		}
	}
	while ($dv =~ s/^(\d+)(\d\d\d)(\D|$)/$1,$2$3/) { ; }

	warn "FIX: $v -> $dv\n" if $d;

	return $du ? sprintf $form, $dv, $du : $dv;
}

sub render_descr ($) {
	my ($d,) = @_;
	$d =~ s/\s+$//g;
	#$d =~ s/[\r\n]+/\n/g;
	$d =~ s/&/\&amp;/g;
	$d =~ s/\r\n/\n/g;
	$d =~ s/\n/\&#13;\&#10;/g;
	$d =~ s/"/\&#34;/g;
	$d =~ s/</\&lt;/g;
	$d =~ s/>/\&gt;/g;
#	$d =~ s/\n/\&#10;/g;
	die "bad descr($1): '$d'\n" if $d =~ /(["<])/;
	return $d;
}

###

my %certrel_by_id;
sub certrel_by_id ($) {
	my ($cid,) = @_;
	die "bad cid '$cid'" unless $cid =~ /^\d+$/;
	&load_certrels() unless %certrel_by_id;
	die "dont have cid $cid" unless $certrel_by_id{$cid};
	return $certrel_by_id{$cid};
}

my %certrels;
sub certrels_by_certid ($) {
	my $cid = shift;
	die "bad classid '$cid'" unless $cid =~ /^\d+$/;
	return @{$certrels{$cid}} if defined $certrels{$cid};
	&load_certrels() unless %certrel_by_id;
	my @cl = ();
	for (keys %certrel_by_id) {
		my $c = $certrel_by_id{$_};
		next unless $c->{childID} == $cid;
		push @cl, $c;
	}
	$certrels{$cid} = \@cl;
	return @{$certrels{$cid}};
}

sub load_certrels {
	return if %certrel_by_id;
#	die "inconsistent" if %types_by_id || %types_by_name;
	my $data = &loadfile("config.BulkData.certificaterelationships");
	for my $row (@$data) {
		my $rid = $row->{relationshipID};
		die "missing rid" unless defined $rid;
		die "trying to redefine certrel $rid" if $certrel_by_id{$rid};
		$certrel_by_id{$rid} = $row;
	}
}


my %certrec_by_id;
sub certrec_by_id ($) {
	my ($cid,) = @_;
	die "bad cid '$cid'" unless $cid =~ /^\d+$/;
	&load_certrecs() unless %certrec_by_id;
	die "dont have cid $cid" unless $certrec_by_id{$cid};
	return $certrec_by_id{$cid};
}

my %certrecs;
sub certrecs_by_certid ($) {
	my $cid = shift;
	die "bad classid '$cid'" unless $cid =~ /^\d+$/;
	return @{$certrecs{$cid}} if defined $certrecs{$cid};
	&load_certrecs() unless %certrec_by_id;
	my @cl = ();
	for (keys %certrec_by_id) {
		my $c = $certrec_by_id{$_};
		next unless $c->{certificateID} == $cid;
		push @cl, $c;
	}
	$certrecs{$cid} = \@cl;
	return @{$certrecs{$cid}};
}

sub load_certrecs {
	return if %certrec_by_id;
#	die "inconsistent" if %types_by_id || %types_by_name;
	my $data = &loadfile("certificateMgr.GetAllShipCertificateRecommendations");
	for my $row (@$data) {
		my $cid = $row->{recommendationID};
		die "missing cid" unless defined $cid;
		die "trying to redefine certrec $cid" if $certrec_by_id{$cid};
		$certrec_by_id{$cid} = $row;
	}
}


my %cert_by_id;
sub cert_by_id ($) {
	my ($cid,) = @_;
	die "bad cid '$cid'" unless $cid =~ /^\d+$/;
	&load_certs() unless %cert_by_id;
	die "dont have cid $cid" unless $cert_by_id{$cid};
	return $cert_by_id{$cid};
}

my %certcats;
sub certs_by_catid ($) {
	my $cid = shift;
	die "bad classid '$cid'" unless $cid =~ /^\d+$/;
	return @{$certcats{$cid}} if defined $certcats{$cid};
	&load_certs() unless %cert_by_id;
	my @cl = ();
	for (keys %cert_by_id) {
		my $c = $cert_by_id{$_};
		next unless $c->{categoryID} == $cid;
		push @cl, $c;
	}
	$certcats{$cid} = \@cl;
	return @{$certcats{$cid}};
}

sub load_certs {
	return if %cert_by_id;
	my $data = &loadfile("config.BulkData.certificates");
	for my $row (@$data) {
		my $cid = $row->{certificateID};
		die "missing cid" unless defined $cid;
		die "trying to redefine cert $cid" if $cert_by_id{$cid};
		$cert_by_id{$cid} = $row;
	}
}

my %certclass_by_id;
sub certclass_by_id ($) {
	my ($cid,) = @_;
	die "bad cid '$cid'" unless $cid =~ /^\d+$/;
	&load_certclasses() unless %certclass_by_id;
	die "dont have cid $cid" unless $certclass_by_id{$cid};
	return $certclass_by_id{$cid};
}

sub load_certclasses {
	return if %certclass_by_id;
	my $data = &loadfile("certificateMgr.GetCertificateClasses");
	for my $row (@$data) {
		my $cid = $row->{classID};
		die "missing cid" unless defined $cid;
		die "trying to redefine cclass $cid" if $certclass_by_id{$cid};
		$certclass_by_id{$cid} = $row;
	}
}


my %certcat_by_id;
sub certcats {
	&load_certcats() unless %certcat_by_id;
	return values %certcat_by_id;
}

sub load_certcats {
	return if %certcat_by_id;
	my $data = &loadfile("certificateMgr.GetCertificateCategories");
	for my $row (@$data) {
		my $cid = $row->{categoryID};
		die "missing cid" unless defined $cid;
		die "trying to redefine ccat $cid" if $certcat_by_id{$cid};
		$certcat_by_id{$cid} = $row;
	}
}
#

sub level_to_grade ($) {
	my $grade = shift;
	die "bad grade $grade" unless $grade =~ /^\d$/;
	my $g = {	1 => "Basic",
			2 => "Standard",
			3 => "Improved",
			4 => "Advanced",
			5 => "Elite",
		}->{$grade};
	die "unknown grade $grade" unless $g;
	return $g;
}



# typeattribs

my %typeattributes;
sub typeattributes ($$) {
	my ($tid,$quiet,) = @_;
	&load_typeattributes() unless %typeattributes;
	unless ($typeattributes{$tid}) {
		warn "dont have attribs for type $tid" unless $quiet;
		return {};
	}
	return $typeattributes{$tid};
}

sub load_typeattributes {
	return if %typeattributes;
	my $data = &loadfile("config.BulkData.dgmtypeattribs");
	for (@$data) {
		my $t = $_->{typeID} || die "missing typeID";
		my $a = $_->{attributeID} || die "missing attributeID";
		my $v = $_->{value};
		$typeattributes{$t} ||= {};
		die "trying to redefine attributeID:$a for typeID:$t" if 
				defined $typeattributes{$t}{$a};
		$typeattributes{$t}{$a} = $v;
	}
}


# typeeffects

my %typeeffects;
sub typeeffects ($$) {
	my ($tid,$quiet,) = @_;
	&load_typeeffects() unless %typeeffects;
	unless ($typeeffects{$tid}) {
		warn "dont have effects for type $tid" unless $quiet;
		return {};
	}
	return $typeeffects{$tid};
}

sub load_typeeffects {
	return if %typeeffects;
	my $data = &loadfile("config.BulkData.dgmtypeeffects");
	for my $row (@$data) {
		my $t = $row->{typeID} || die "missing typeID";
		my $a = $row->{effectID} || die "missing effectID";
		my $v = $row->{isDefault};
		$typeeffects{$t} ||= {};
		die "trying to redefine effectID:$a for typeID:$t" if 
				defined $typeeffects{$t}{$a};
		$typeeffects{$t}{$a} = $v;
	}
}


# types

my %types_by_id;
sub type_by_id ($) {
	my ($cid,) = @_;
	die "bad type '$cid'" unless $cid =~ /^\d+$/;
	&load_types() unless %types_by_id;
	die "dont have type $cid" unless $types_by_id{$cid};
	return $types_by_id{$cid};
}

sub type_ids {
	&load_types() unless %types_by_id;
	return keys %types_by_id;
}

my %types_by_name;
sub type_by_name ($) {
	my ($cname,) = @_;
	&load_types() unless %types_by_name;
	die "dont have type $cname" unless $types_by_name{$cname};
	return $types_by_name{$cname};
}

my %grouptypes;
sub types_by_group ($) {
	my ($gid,) = @_;
	unless ($gid =~ /^\d+$/) {
		printf "Resolving group '%s' ...", $gid;
		my $c = &group_by_name($gid);
		$gid = $c->{groupID};
		printf " %i\n", $gid;
	}
	return @{$grouptypes{$gid}} if defined $grouptypes{$gid};
	my @types = ();
	for (keys %types_by_id) {
		next unless $types_by_id{$_}->{groupID} == $gid;
		push @types, $types_by_id{$_};
	}
	$grouptypes{$gid} = \@types;
	return @{$grouptypes{$gid}};
}


sub load_types {
	return if %types_by_id && %types_by_name;
	die "inconsistent" if %types_by_id || %types_by_name;
	my $data = &loadfile("config.BulkData.types");
	for my $row (@$data) {
		my $cid = $row->{typeID};
		die "missing cid" unless defined $cid;
		die "trying to redefine type $cid" if $types_by_id{$cid};
		$types_by_id{$cid} = $row;
		my $cname = $row->{typeName};
		if ($types_by_name{$cname}) {
			warn "trying to redefine type $cname";
			next;
		}
		$types_by_name{$cname} = $row;
	}
}

# metatypes

my %variations_by_id;
sub variations_by_id ($) {
	my ($id,) = @_;
	&load_variations() unless %variations_by_id;
	return () unless $variations_by_id{$id};
#	die "dont have variation $id" unless $variations_by_id{$id};
	return keys %{$variations_by_id{$id}};
}

my %metagroup_by_typeid;
sub metagroup_by_typeid ($) {
	my ($id,) = @_;
	&load_variations() unless %metagroup_by_typeid;
	my $mid = $metagroup_by_typeid{$id};
	return undef unless $mid;
	return &metagroup_by_id($mid);
#	die "dont have variation $id" unless $variations_by_id{$id};
}

sub load_variations {
	return if %variations_by_id;
	my $data = &loadfile("config.BulkData.invmetatypes");
	for my $row (@$data) {
		my $tid = $row->{typeID};
		die "missing tid" unless defined $tid;
		my $pid = $row->{parentTypeID};
		die "missing pid" unless defined $pid;

		$variations_by_id{$tid} ||= {};
		$variations_by_id{$tid}{$pid}++;
		$variations_by_id{$pid} ||= {};
		$variations_by_id{$pid}{$tid}++;
		$metagroup_by_typeid{$tid} = $row->{metaGroupID};
	}
}


# group

my %groups_by_id;
sub group_by_id ($) {
	my ($cid,) = @_;
	&load_groups() unless %groups_by_id;
	die "dont have group $cid" unless $groups_by_id{$cid};
	return $groups_by_id{$cid};
}

my %groups_by_name;
sub group_by_name ($) {
	my ($cname,) = @_;
	&load_groups() unless %groups_by_name;
	die "dont have group $cname" unless $groups_by_name{$cname};
	return $groups_by_name{$cname};
}

my %catgroups;
sub groups_by_category ($) {
	my ($cid,) = @_;
	unless ($cid =~ /^\d+$/) {
		printf "Resolving category '%s' ...", $cid;
		my $c = &category_by_name($cid);
		$cid = $c->{categoryID};
		printf " %i\n", $cid;
	}
	return @{$catgroups{$cid}} if defined $catgroups{$cid};
	my @groups = ();
	for (keys %groups_by_id) {
		next unless $groups_by_id{$_}->{categoryID} == $cid;
		push @groups, $groups_by_id{$_};
	}
	$catgroups{$cid} = \@groups;
	return @{$catgroups{$cid}};
}

sub load_groups {
	return if %groups_by_id && %groups_by_name;
	die "inconsistent" if %groups_by_id || %groups_by_name;
	my $data = &loadfile("config.BulkData.groups");
	for my $row (@$data) {
		my $cid = $row->{groupID};
		die "missing cid" unless defined $cid;
		die "trying to redefine group $cid" if $groups_by_id{$cid};
		$groups_by_id{$cid} = $row;
		my $cname = $row->{groupName};
		die "trying to redefine group $cname" if $groups_by_name{$cname};
		$groups_by_name{$cname} = $row;
	}
}

# attributes

my %attributes_by_id;
sub attribute_by_id ($) {
	my ($cid,) = @_;
	&load_attributes() unless %attributes_by_id;
	die "dont have attribute $cid" unless $attributes_by_id{$cid};
	return $attributes_by_id{$cid};
}

my %attributes_by_name;
sub attribute_by_name ($) {
	my ($cname,) = @_;
	&load_attributes() unless %attributes_by_name;
	die "dont have attribute $cname" unless $attributes_by_name{$cname};
	return $attributes_by_name{$cname};
}

sub load_attributes {
	return if %attributes_by_id && %attributes_by_name;
	die "inconsistent" if %attributes_by_id || %attributes_by_name;
	my $data = &loadfile("config.BulkData.dgmattribs");
	for my $row (@$data) {
		my $cid = $row->{attributeID};
		die "missing cid" unless defined $cid;
		die "trying to redefine attribute $cid" if $attributes_by_id{$cid};
		$attributes_by_id{$cid} = $row;
		my $cname = $row->{attributeName};
		warn "trying to redefine attribute $cname" if $attributes_by_name{$cname};
		$attributes_by_name{$cname} = $row;
	}
}

# marketgroups

my (%marketgroups_by_id, %marketgroups_by_name,);
sub marketgroup_by_id ($) {
	my ($cid,) = @_;
	&load_marketgroups() unless %marketgroups_by_id;
	confess "dont have marketgroup $cid" unless $marketgroups_by_id{$cid};
	return $marketgroups_by_id{$cid};
}

sub marketgroup_by_name ($) {
	my ($cname,) = @_;
	&load_marketgroups() unless %marketgroups_by_name;
	die "dont have marketgroup $cname" unless $marketgroups_by_name{$cname};
	return $marketgroups_by_name{$cname};
}

sub marketgroup_ids {
	&load_marketgroups() unless %marketgroups_by_id;
	return keys %marketgroups_by_id;
}

sub load_marketgroups {
	return if %marketgroups_by_id;
	die "inconsistent: name without id" if %marketgroups_by_name;
	my $data = &loadfile("market.GetMarketGroups");
	my @header = @{$data->{header}};
	my $d = $data;
	$d = $d->{'machostring (id:40)'} || $d->{'items'};
	die "no marketgroups" unless $d && ref $d eq 'HASH';
	for (keys %$d) {
		for (@{$d->{$_}}) {
			my @row = @$_;
			die "col count mismatch" unless scalar(@header) == scalar(@row);
			my %row=();
			for (0..$#row) {
				my $k = $header[$_];
				my $v = $row[$_];
				die "trying to redefine key $k" if $row{$k};
				$row{$k} = $v;
			}
			my $cid = $row{marketGroupID};
			die "missing cid" unless defined $cid;
			die "trying to redefine marketgroup $cid" if $marketgroups_by_id{$cid};
			$marketgroups_by_id{$cid} = \%row;

			my $cname = $row{marketGroupName};
			die "missing cname" unless defined $cname;
			next if $marketgroups_by_name{$cname};
			warn "WARN: trying to redefine marketgroup $cname" if $marketgroups_by_name{$cname};
			$marketgroups_by_name{$cname} = \%row;
		}
	}
#	print Dumper( \%marketgroups_by_id );
}

# graphics

my %graphics_by_id;
sub graphic_by_id ($) {
	my ($cid,) = @_;
	&load_graphics() unless %graphics_by_id;
	die "dont have graphic $cid" unless $graphics_by_id{$cid};
	return $graphics_by_id{$cid};
}

sub load_graphics {
	return if %graphics_by_id;
	my $data = &loadfile("config.BulkData.graphics");
	for my $row (@$data) {
		my $cid = $row->{graphicID};
		die "missing cid" unless defined $cid;
		die "trying to redefine graphic $cid" if $graphics_by_id{$cid};
		$graphics_by_id{$cid} = $row;
	}
}

# categories

my %categories_by_id;
sub category_by_id ($) {
	my ($cid,) = @_;
	&load_categories() unless %categories_by_id;
	die "dont have category $cid" unless $categories_by_id{$cid};
	return $categories_by_id{$cid};
}

my %categories_by_name;
sub category_by_name ($) {
	my ($cname,) = @_;
	&load_categories() unless %categories_by_name;
	die "dont have category $cname" unless $categories_by_name{$cname};
	return $categories_by_name{$cname};
}

sub load_categories {
	return if %categories_by_id && %categories_by_name;
	die "inconsistent" if %categories_by_id || %categories_by_name;
	my $data = &loadfile("config.BulkData.categories");
	for my $row (@$data) {
		my $cid = $row->{categoryID};
		die "missing cid" unless defined $cid;
		die "trying to redefine category $cid" if $categories_by_id{$cid};
		$categories_by_id{$cid} = $row;
		my $cname = $row->{categoryName};
		die "trying to redefine category $cname" if $categories_by_name{$cname};
		$categories_by_name{$cname} = $row;
	}
}

### metagroups

my %metagroups_by_id;
sub metagroup_by_id ($) {
	my ($id,) = @_;
	&load_metagroups() unless %metagroups_by_id;
	die "dont have metagroup $id" unless $metagroups_by_id{$id};
	return $metagroups_by_id{$id};
}

sub load_metagroups {
	return if %metagroups_by_id;
	my $data = &loadfile("config.BulkData.metagroups");
	for my $row (@$data) {
		my $cid = $row->{metaGroupID};
		die "missing cid" unless defined $cid;
		die "trying to redefine metagroup $cid" if $metagroups_by_id{$cid};
		$metagroups_by_id{$cid} = $row;
	}
}

### units

my %units_by_id;
sub unit_by_id ($) {
	my ($uid,) = @_;
	&load_units() unless %units_by_id;
	die "dont have unit $uid" unless $units_by_id{$uid};
	return $units_by_id{$uid};
}

sub load_units {
	return if %units_by_id;
	my $data = &loadfile("config.BulkData.units");
	die "no data" unless $data;
	die "data not array: $data" unless ref $data eq 'ARRAY';
	for my $row (@$data){
		my $cid = $row->{unitID};
		die "missing cid" unless defined $cid;
		die "trying to redefine unit $cid" if $units_by_id{$cid};
		$units_by_id{$cid} = $row;
	}
}

###

sub loadfile ($) {
	my ($fn,) = @_;
	printf "FILE: %s ... ", $fn;
	for my $efn ($fn, "$fn.dump", "$datadir/$fn", "$datadir/$fn.dump") {
		next unless -e $efn;
		printf "%s ... ", $efn;
		my $cd = do $efn or die "cant do '$efn': '$!'";
		die "undef cd" unless defined $cd;
		die "no cd" unless $cd;
		printf "%s ... ", $cd;

	my $data = $cd;
	if (ref $data eq 'ARRAY' && ref $data->[0] eq 'ARRAY') {
		my @header = @{$data->[0]};
		my $d = [];
		for (@{$data->[1]}) {
			my @row = @$_;
			die "col count mismatch" unless scalar(@header) == scalar(@row);
			my %row=();
			for (0..$#row) {
				my $k = $header[$_];
				my $v = $row[$_];
				die "trying to redefine key $k" if $row{$k};
				$row{$k} = $v;
			}
			push @$d, \%row;
		}
		$data = $d;
	} elsif (ref $data eq 'HASH') {
		die "no data" unless $data->{data};
		$data = $data->{data};
	}

		printf "%s ... done\n", $data;
		return $data;
	}
	die "not found";
}

sub slurpfile ($) {
	my ($fn,) = @_;
	open IF, $fn or die "cant open '$fn': $!";
	undef $/;
	my $data = <IF>;
	close IF or die "cant close '$fn': $!";
	return $data;
}