CPAN::Groonga

a non-issue.

=head2 Storage Formats

The storage format is supposed to be platform-independent.  JSON seems like a
good default encoding, however it requires strings to be in Unicode.  When you
encod

ll which were which.  This means that
if you take a unicode-as-octets filename and encode it with JSON and decode it
again, perl will mangle it when you attempt to open the file, and fail.  It
also me

names
which are not valid UTF-8 are encoded as structures which can be restored
when decoding the JSON.

=head2 Conclusion

In the end, I came up with a module called L<DataStore::CAS::FS::InvalidUTF8

Store::CAS::FS::DirCodec::Unix;
use 5.008;
use strict;
use warnings;
use Try::Tiny;
use Carp;
use JSON 2.53 ();
use Scalar::Util 'looks_like_number';
require DataStore::CAS::FS::Dir;
require DataStore

y encode only the attributes of a UNIX stat()


our $_json_coder;
sub _build_json_coder {
	DataStore::CAS::FS::InvalidUTF8->add_json_filter(
		JSON->new->utf8->canonical->convert_blessed, 1
	);
}

our

{_}{gmap}= \%gmap;
	
	my $meta_json= ($_json_coder ||= _build_json_coder())->encode($metadata);
	my $ret= "CAS_Dir 04 unix\n"
		.pack('N', length($meta_json)).$meta_json
		.join('', sort { substr($a,4

end I decided on a
pluggable implementation, where a "Universal" plugin uses something plain like
JSON, and more specialized plugins do things like storing an array of UNIX
'stat' fields.  Users can a

em Universal

L<DataStore::CAS::FS::DirCodec::Universal> stores all metadata of each DirEnt
using JSON.  If you use this codec, you are guaranteed that anything your
CAS::FS::Scanner picked up was sav

S::FS::DirCodec::Universal;
use 5.0080001;
use strict;
use warnings;
use Try::Tiny;
use Carp;
use JSON 2.53 ();
require DataStore::CAS::FS::Dir;
require DataStore::CAS::FS::DirEnt;
require DataStore::

c for saving all arbitrary fields of a DirEnt


our $_json_coder;
sub _build_json_coder {
	DataStore::CAS::FS::InvalidUTF8->add_json_filter(
		JSON->new->utf8->canonical->convert_blessed, 1
	);
}

sub

entry: ".JSON::encode_json($entry);
			defined $entry->{type} or croak "Can't serialize typeless directory entry: ".JSON::encode_json($entry);
			!defined($_) || (ref $_? ref($_)->can("TO_JSON") : &ut

:CAS::FS::DirCodec::Minimal;
use 5.008001;
use strict;
use warnings;
use Try::Tiny;
use Carp;
use JSON 2.53 ();
require DataStore::CAS::FS::InvalidUTF8;
require DataStore::CAS::FS::Dir;
*decode_utf8= 

$entry_list;
	
	my $ret= "CAS_Dir 00 \n";
	if ($metadata and scalar keys %$metadata) {
		my $enc= JSON->new->utf8->canonical->convert_blessed;
		$ret .= $enc->encode($metadata);
	}
	$ret .= "\0";
	$re

ing end of metadata";
	if ($meta_end > 0) {
		my $enc= JSON->new()->utf8->canonical->convert_blessed;
		DataStore::CAS::FS::InvalidUTF8->add_json_filter($enc);
		$params->{metadata}= $enc->decode(subs

f : $$self.$other);
}


sub add_json_filter {
	my ($self, $json)= @_;
	$json->filter_json_single_key_object(
		'*InvalidUTF8*' => \&FROM_JSON
	);
	$json;
}

sub TO_JSON {
	my $x= ${$_[0]};
	utf8::upgr

ade($x);
	return { '*InvalidUTF8*' => $x };
}

sub FROM_JSON {
	my $x= $_[0];
	utf8::downgrade($x) if utf8::is_utf8($x);
	return bless \$x, __PACKAGE__;
}

sub TO_UTF8 {
	${$_[0]};
}

1;

__END__

=po

 SYNOPSIS

  my $j= JSON->new()->convert_blessed;
  DataStore::CAS::FS::InvalidUTF8->add_json_filter($j);
  my $x= DataStore::CAS::FS::InvalidUTF8->decode_utf8("\x{FF}");
  my $json= $j->encode($x);