HITCON CTF 2019 Quals: Reverse - EmojiVM

enum OpCode { NOP, ADD, SUB, MUL, MOD, XOR, AND, IS_LESS, IS_EQ, JMP, JMP_IF, JMP_IF_FALSE,
              PUSH_EMOJI, POP, LD, ST, NEW, FREE, READ, POP_OBJ, FLUSH, POP_INT64, EXIT, 
              NUM_0, NUM_1, NUM_2, NUM_3, NUM_4, NUM_5, NUM_6, NUM_7, NUM_8, NUM_9, NUM_10,
              PUSH_VAL, STORE_OBJ_VAL };

class Instruction
{
    public OpCode OpCode { get; set; }
    public long Value { get; set; }
    public int FileOffset { get; set; }
    public int InstructionIdx { get; set; }
    public bool IsNum => OpCode.NUM_0 <= OpCode && OpCode <= OpCode.NUM_10;
    public bool IsPush => OpCode == OpCode.PUSH_EMOJI || OpCode == OpCode.PUSH_VAL;

    public int ObjectIdx { get; set; }
    public int ByteIdx { get; set; }
    public bool IsStore => OpCode == OpCode.ST;

    public override string ToString() => 
        OpCode == OpCode.PUSH_VAL ? $"PUSH {Value}" : 
        OpCode == OpCode.STORE_OBJ_VAL ? $"obj{ObjectIdx}[{ByteIdx}] = {Value}" : 
        $"{OpCode}";

    public static Instruction Push(long argument, Instruction orig = null) => 
        new Instruction() { OpCode = OpCode.PUSH_VAL, Value = argument, FileOffset = orig?.FileOffset ?? 0, 
            InstructionIdx = orig?.InstructionIdx ?? 0 };
    public static Instruction Store(int objectIdx, int byteIdx, long value) => 
        new Instruction() { OpCode = OpCode.STORE_OBJ_VAL, ObjectIdx = objectIdx, ByteIdx = byteIdx, Value = value };
}

class InstrFilter
{
    public int SourceLen { get; set; }
    public Func<Instruction[], bool> Matcher { get; set; }
    public Func<Instruction[], Instruction> Converter { get; set; }

    public InstrFilter(int sourceLen, Func<Instruction[], bool> tester, Func<Instruction[], Instruction> converter)
    {
        SourceLen = sourceLen;
        Matcher = tester;
        Converter = converter;
    }
}

static void EmojiDecode()
{
    var baseDir = @"g:\Dropbox\hack\hitcon19\challs\emojivm_reverse\";
    var opCodes = File.ReadAllLines($"{baseDir}opcodes_enc.txt").Select(
          (codePoint, i) => new { codePoint, opCode = (OpCode)i }).ToArray();

    var prgSrc = File.ReadAllText($"{baseDir}chal.evm");
    var prg0 = new List<Instruction>();
    for (int i = 0; i < prgSrc.Length;)
    {
        var opCodeInfo = opCodes.Single(x => prgSrc.Substring(i, x.codePoint.Length) == x.codePoint);
        prg0.Add(new Instruction() { FileOffset = i, InstructionIdx = prg0.Count, OpCode = opCodeInfo.opCode });
        i += opCodeInfo.codePoint.Length;
    }

    Instruction[] RunFilters(Instruction[] source, InstrFilter[] filters)
    {
        var tail = new Instruction[10];
        for (int i = 0; i < source.Length;)
        {
            var hadMatch = false;
            foreach (var filter in filters)
            {
                var filterInput = source.Skip(i).Concat(tail).Take(filter.SourceLen).ToArray();
                if (filter.Matcher(filterInput))
                {
                    source = source.Take(i).Concat(new[] { filter.Converter(filterInput) })
                        .Concat(source.Skip(i + filter.SourceLen)).ToArray();
                    hadMatch = true;
                    break;
                }
            }

            if (!hadMatch)
                i++;
        }
        return source;
    }

    Instruction[] RunFiltersMulti(Instruction[] source, InstrFilter[] filters)
    {
        while (true)
        {
            var oldSourceLen = source.Length;
            source = RunFilters(source, filters);
            if (source.Length == oldSourceLen)
                break;
        }
        return source;
    }

    var prg1 = RunFiltersMulti(prg0.ToArray(), new[]
    {
        new InstrFilter(2, ins => ins[0].IsPush && ins[1].IsNum, 
            ins => Instruction.Push(ins[1].OpCode - OpCode.NUM_0, ins[0])),

        new InstrFilter(3, ins => ins[0].IsPush && ins[1].IsPush && ins[2].OpCode == OpCode.MUL, 
            ins => Instruction.Push(ins[0].Value * ins[1].Value, ins[0])),

        new InstrFilter(3, ins => ins[0].IsPush && ins[1].IsPush && ins[2].OpCode == OpCode.ADD,
            ins => Instruction.Push(ins[0].Value + ins[1].Value, ins[0])),
    });

    var prg2 = RunFiltersMulti(prg1, new[] { new InstrFilter(4, 
          ins => ins[0].IsPush && ins[1].IsPush && ins[2].IsPush && ins[3].OpCode == OpCode.ST, 
          ins => Instruction.Store((int)ins[2].Value, (int)ins[1].Value, ins[0].Value)) });

    var prg1src = prg1.Select(x => $"{x.InstructionIdx}: {x}").ToArray();
    File.WriteAllLines($"{baseDir}prg1src.txt", prg1src);
}

var flagKey = EncodingHelper.GetBytes("\x18\x05\x1d\x10\x42\x09\x4a\x24\x00\x5b\x08\x17\x40\x00\x72\x30\x09\x6c\x56\x40\x09\x5b\x05\x1a\x00");
var toMatch = EncodingHelper.GetBytes("\x8e\x63\xcd\x12\x4b\x58\x15\x17\x51\x22\xd9\x04\x51\x2c\x19\x15\x86\x2c\xd1\x4c\x84\x2e\x20\x06\x00");
var input = toMatch.Select((x,i) => (byte)(
    i % 4 == 0 ? x - 30 : 
    i % 4 == 1 ? (x ^ 7) + 8 : 
    i % 4 == 2 ? ((x + 4) ^ 68) - 44 :
    x ^ 101 ^ (172 & 20))).ToArray();
var flag = EncodingHelper.GetString(CryptoUtils.XorEqual(flagKey, input));