外字集計

REM 外字集計.bat

ruby -x "%~f0"
goto end

#!ruby
#encoding: utf-8
Encoding.default_external = 'UTF-8'

$stdout=open("allkanji.txt","w")
$stderr=open("err.txt","w")

#漢字実体(UTF-8),Unicode,JIS区点コード,出現数,出現作品数

h=open(__dir__+"/aozora_gaiji.txt").readlines.
  map{|e|e.chomp.split(/(?<!,),/)}.to_h
h.transform_keys!{|k| k.split("-").map{|f|f.to_i}}

a1,a2,hs=[],[],{}
Dir.glob('**/*.txt'){|f|
  next if /aozora_gaiji.txt$|err.txt$|allkanji.txt$/=~f
  i=0
  begin
    s=open(f).read.encode(Encoding::UTF_8,Encoding::SJIS).
      scan(%r|※［＃[^［］]*］|).map{|e|
        if /((\d)-(\d+)-(\d+))/=~e
          s=$1
          n=s.split("-").map{|k| k.to_i}
          if h[n]
            e=h[n]
            if hs.key?(e)
              hs[e][2]+=1
              hs[e][3]+=1 if i==0
              i=1
            else
              hs[e]=["U+"+e.ord.to_s(16).tr('a-z','A-Z'),s,1,1]
           end
          else
            a1 << [f,e]
          end
        end
    }
  rescue
    $stderr.puts "error:"+f
    next
  end
}
a1.uniq.each{|e| $stderr.puts e.join("\n")}
hs.each{|k,v| a2 << [k,v].flatten}
a2.sort_by{|e| e[1]}.each{|e| puts e.join(",")}
__END__
:end