package com.jaytux.simd.data import com.fleeksoft.ksoup.Ksoup import com.fleeksoft.ksoup.parser.Parser import com.jaytux.simd.data.IntrinsicInstructions.xed import kotlinx.coroutines.coroutineScope import kotlinx.datetime.* import kotlinx.datetime.format.MonthNames import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import org.jetbrains.exposed.sql.batchInsert import org.jetbrains.exposed.sql.insert import org.jetbrains.exposed.sql.transactions.transaction import java.io.File import org.json.JSONObject import org.junit.jupiter.api.DisplayNameGenerator.Simple import java.text.DateFormat import java.text.SimpleDateFormat import java.util.Date import java.util.prefs.Preferences object Loader { private val cache = Preferences.userNodeForPackage(this::class.java) private val intelDateTimeFormat = LocalDate.Format { monthNumber(); chars("/"); dayOfMonth(); chars("/"); year(); } private val dateFormat = LocalDate.Format { dayOfMonth(); chars("-"); monthName(MonthNames.ENGLISH_FULL); chars("-"); year() } private val zero = LocalDate(1970, 1, 1) data class XmlIntrinsic(val name: String, val tech: String, val retType: String, val retVar: String?, val args: List>, val desc: String, val op: String?, val insn: List>, val cpuid: String?, val category: String ) data class Version(val major: Int, val minor: Int, val patch: Int) { override fun toString(): String = "$major.$minor.$patch" infix fun isNewerThan(other: Version): Boolean { if(major > other.major) return true if(major == other.major && minor > other.minor) return true if(major == other.major && minor == other.minor && patch > other.patch) return true return false } companion object { fun fromString(version: String): Version { val parts = version.split(".") return Version(parts[0].toInt(), parts[1].toInt(), parts[2].toInt()) } } } data class DatedVersion(val version: Version, val releaseDate: LocalDate, val updateDate: LocalDate) { override fun toString(): String = "$version (remote released ${dateFormat.format(releaseDate)}; local updated ${dateFormat.format(updateDate)})" companion object { fun fromPrefs(): DatedVersion { val v = cache.get("version", null)?.let { Version.fromString(it) } ?: Version(0, 0, 0) val r = cache.get("release", null)?.let { dateFormat.parse(it) } ?: zero val u = cache.get("update", null)?.let { dateFormat.parse(it) } ?: zero return DatedVersion(v, r, u) } } } data class XmlData( val version: DatedVersion, val types: Set, val techs: Set, val cpuids: Set, val categories: Set, val intrinsics: List ) data class Performance(val latency: Float?, val throughput: Float?) data class JsonData(val platforms: Set, val data: Map>) private suspend fun updateVersion(version: DatedVersion) = coroutineScope { cache.put("version", version.version.toString()) cache.put("release", dateFormat.format(version.releaseDate)) cache.put("update", dateFormat.format(version.updateDate)) } suspend fun loadXml(xmlFile: String): XmlData = coroutineScope { val xml = Ksoup.parseXml(File(xmlFile).readText(Charsets.UTF_8)) Parser.xmlParser() val cppTypes = mutableSetOf() val techs = mutableSetOf() val cpuids = mutableSetOf() val categories = mutableSetOf() val intrins = mutableListOf() val errors = mutableListOf() val (version, release) = xml.getElementsByTag("intrinsics_list").firstOrNull()?.let { val version = it.attribute("version")?.value?.let { v -> Version.fromString(v) } if(version == null) { errors += "Missing version attribute in intrinsics_list element" return@let null } val date = it.attribute("date")?.value?.let { d -> intelDateTimeFormat.parse(d) } if(date == null) { errors += "Missing release_date attribute in intrinsics_list element" return@let null } version to date } ?: Version(0, 0, 0) to LocalDate.fromEpochDays(0) // dummy data if(errors.isNotEmpty()) { errors.forEach { System.err.println(it) } throw Exception("XML file is (partially) invalid") } val update = Clock.System.now().toLocalDateTime(TimeZone.currentSystemDefault()).date xml.getElementsByTag("intrinsic").forEachIndexed { i, it -> val name = it.attribute("name")?.value if(name == null) { errors += "Missing name attribute in intrinsic element" return@forEachIndexed } val tech = it.attribute("tech")?.value if(tech == null) { errors += "Missing tech attribute for intrinsic $name" return@forEachIndexed } val ret = it.getElementsByTag("return").firstOrNull() if(ret == null) { errors += "Missing return element for intrinsic $name" return@forEachIndexed } val retType = ret.attribute("type")?.value if(retType == null) { errors += "Missing type attribute for return element in intrinsic $name" return@forEachIndexed } val retVar = ret.attribute("varname")?.value val args = mutableListOf>() it.getElementsByTag("parameter").forEachIndexed { i, p -> val argName = p.attribute("varname")?.value val type = p.attribute("type")?.value if(type != null && type == "void") return@forEachIndexed //ignore if(argName == null) { errors += "Missing varname attribute for parameter $i in intrinsic $name" return@forEachIndexed } if(type == null) { errors += "Missing type attribute for parameter $argName in intrinsic $name" return@forEachIndexed } cppTypes += type args += argName to type } val desc = it.getElementsByTag("description").firstOrNull()?.wholeText()?.trim() if(desc == null) { errors += "Missing description element for intrinsic $name" return@forEachIndexed } val op = it.getElementsByTag("operation").firstOrNull()?.wholeText()?.trim() val insn = mutableListOf>() it.getElementsByTag("instruction").forEachIndexed { i, ins -> val insnName = ins.attribute("xed")?.value ?: ins.attribute("name")?.value if(insnName == null) { errors += "Missing both xed and name attribute for instruction $i in intrinsic $name" return@forEachIndexed } val insnMnemonic = ins.attribute("name")?.value if(insnMnemonic == null) { errors += "Missing name attribute for instruction $insnName in intrinsic $name" return@forEachIndexed } val insnForm = ins.attribute("form")?.value insn += Triple(insnName, insnMnemonic, insnForm) } val cpuid = it.getElementsByTag("cpuid").firstOrNull()?.text() val category = it.getElementsByTag("category").firstOrNull()?.text() if(category == null) { errors += "Missing category element for intrinsic $name" return@forEachIndexed } val intrinsic = XmlIntrinsic(name, tech, retType, retVar, args, desc, op, insn, cpuid, category) intrins += intrinsic techs += tech cpuid?.let { c -> cpuids += c } categories += category cppTypes += retType } if(errors.isNotEmpty()) { errors.forEach { System.err.println(it) } throw Exception("XML file is (partially) invalid") } XmlData(version = DatedVersion(version, release, update), types = cppTypes, techs = techs, cpuids = cpuids, categories = categories, intrinsics = intrins ) } suspend fun loadJson(jsonFile: String): JsonData = coroutineScope { val json = File(jsonFile).readText(Charsets.UTF_8) val schema = JSONObject(json) val pSet = mutableSetOf() val res = mutableMapOf>() schema.keys().forEach { opcode -> val pMap = mutableMapOf() val platforms = schema.getJSONArray(opcode) for (i in 0 until platforms.length()) { val platform = platforms.getJSONObject(i) platform.keys().forEach { k -> pSet += k val latency = platform.getJSONObject(k).getString("l").toFloatOrNull() val throughput = platform.getJSONObject(k).getString("t").toFloatOrNull() pMap += k to Performance(latency, throughput) } } res += opcode to pMap } JsonData(pSet, res) } suspend fun importToDb(xml: XmlData, json: JsonData) = coroutineScope { val db = Database.db transaction { val techMap = xml.techs.associateWith { tech -> Tech.new { name = tech } } val typeMap = xml.types.associateWith { type -> CppType.new { name = type } } val catMap = xml.categories.associateWith { cat -> Category.new { name = cat } } val cpuidMap = xml.cpuids.associateWith { cpuid -> CPUID.new { name = cpuid } } val platformMap = json.platforms.associateWith { platform -> Platform.new { name = platform } } xml.intrinsics.forEach { intr -> val dbIn = Intrinsic.new { mnemonic = intr.name returnType = typeMap[intr.retType] ?: throw Exception("Type ${intr.retType} not found") returnVar = intr.retVar description = intr.desc operations = intr.op category = catMap[intr.category] ?: throw Exception("Category ${intr.category} not found") cpuid = intr.cpuid?.let { cpuidMap[it] ?: throw Exception("CPUID ${intr.cpuid} not found") } tech = techMap[intr.tech] ?: throw Exception("Tech ${intr.tech} not found") } intr.args.forEachIndexed { i, arg -> IntrinsicArgument.new { intrinsic = dbIn name = arg.first type = typeMap[arg.second] ?: throw Exception("Type ${arg.second} not found") index = i } } intr.insn.forEach { insn -> val dbInsn = IntrinsicInstruction.new { intrinsic = dbIn xed = insn.first mnemonic = insn.second insn.third?.let { form = it } } json.data[insn.first]?.forEach { (pl, perf) -> val dbPl = platformMap[pl] ?: throw Exception("Platform $pl not found") Performances.insert { it[instruction] = dbInsn.id it[platform] = dbPl.id it[latency] = perf.latency it[throughput] = perf.throughput } } } } } updateVersion(xml.version) } }