User:Amgine/Dump processing/test xml.php

<?php
/*
 * test_xml.php
 *
 * Copyright 2014 Amgine <amgine@Amgines-MacBook-Air.local>
 *
 * This program is free software. It comes without any warranty, to the extent
 * permitted by applicable law. You can redistribute it and/or modify it under
 * the terms of the Do What The Fuck You Want To Public License, Version 2, as
 * published by Sam Hocevar. See http://www.wtfpl.net/ for more details.
 *
 */

/**
 * function: getL2
 **
 * Return an array of L2 headers from a wikitext string.
 **
 * @param string $text The wiki syntax string of the page.
 * @return array An array of language names (or other level 2 header texts.)
 **/
function getL2( $text ){
	$l2 = array();

	foreach( explode( "\n", $text ) as $line ){
		$matches = array();
		if( preg_match_all( '/^==([\w\s\-‒-—‑¯­_]+)==/u', $line, $matches ) ){
			$l2[] = trim( $matches[1][0] );
		}
	}
	if( count( $l2 ) > 0 ){
		return $l2;
	}
}

/**
 * function: add2Dictionary
 **
 * Process found term (breaking up phrases), check for uniqueness, and
 * add to the dictionary.
 **
 * @param string $term The found term to be added to the dictionary.
 * @param array $dictionary The dictionary which will be appended.
 **/
function add2Dictionary( $term, &$dictionary, $lang, $whitespace = false ){
	if( $whitespace ){
		if( !in_array( trim( $term ), $dictionary ) ){
			$dictionary[] = trim( $term );
		}
	}else{
		if( preg_match( '/[\w]+/', $term ) ){
			$term = explode( ' ', $term );
			foreach( $term as $word ){
				if( !in_array( trim( $word ), $dictionary ) ){
					$dictionary[] = trim( $word );
				}
			}
		}
	}
	// Buffer 500 entries in ram per language; a reasonable performance vs. memory explode
	if( count( $dictionary ) > 500 ){
		if( !file_put_contents( $lang . '.txt', implode( "\n", $dictionary ), FILE_APPEND ) ){
			die( "Writing $lang dictionary -- FAILED!\n" );
		}
		$dictionary = array();
	}
}

$reader = new XMLReader();

// @FIXME: hard-coded dump filename
if( !$reader->open( '/Volumes/VERBATIM HD/enwiktionary-20140311-pages-meta-current.xml' ) ){
	die( "Failed to open file for xml reading.\n" );
}

// @FIXME: evil, vile, but it works
$reader->read(); $reader->read();
$reader->next( 'page' );

$dictionary = array();

# Try Unicode magic to check for: MIXED_SCRIPT_CONFUSABLE, SINGLE_SCRIPT, INVISIBLE
# https://ssl.icu-project.org/apiref/icu4c/uspoof_8h.html#a0dbd60e53a571689baf65c63f4de8155
# @FIXME: Before this, remove words not in the main script of the language (requires name->code)
# @FIXME: Currently stripping all non-Latin!
$checker = new Spoofchecker();
$checker->setChecks ( 50 );

$i = 0;
do{
	// Grab <page> node, make simple xml object of it.
	$node = $reader->readOuterXML();
	$element = simplexml_load_string( $node );
	if( $element->ns == '0' && !$checker->isSuspicious( $element->title ) ){
		// If in main namespace, and not confusable, grab L2 headers
		// @FIXME: Create a smarter exclusionary system
		// @TODO: Figure out what the consensus is on proper names
		if( !preg_match( '/\{\{context\|[^}]*\b(vulgar(?:ity?)\b|obscen(?:e|ity)|offensive|pejorative|medicine|slang)[^}]*\}\}/iu', $element->revision->text ) ){
			$l2s = getL2( $element->revision->text );
			if( is_array( $l2s ) ){
				foreach( $l2s as $lang ){
					if( !array_key_exists( $lang, $dictionary ) ){
						$dictionary[$lang] = array();
					}
					// NB: Optional $whitespace = true for function allows multi-word phrases,
					// default does not. '''Default may create broken CJK phrases'''
					add2Dictionary( $element->title, $dictionary[$lang], $lang );
				}
			}
		} else {
			add2Dictionary( $element->title, $dictionary['blacklist'], 'blacklist', $whitespace = true );
		}
	}
	++$i;
	echo "\r$i";
}while( $reader->next( 'page' ) );

// File is parsed, flush buffered dictionaries
echo "\n\n";
foreach( $dictionary as $lang => $dict ){
	echo "Writing $lang dictionary.";
	if( file_put_contents( $lang . '.txt', implode( "\n", $dictionary[$lang] ), FILE_APPEND ) ){
		echo "\rWrote $lang dictionary successfully.\n";
	}else{
		echo " -- FAILED!\n";
	}
}

// @FIXME: move langname dictionaries to langcode dictionaries.s