Code: Select all
#!/usr/bin/env python
# -*- coding: utf-8 -*-
r"""slkfix.py - make an old "SLK mode" map load again on Warcraft III 3.0.0 (build 24268+).
Patch 3.0.0 breaks maps made with KKWE or YDWE, and any map that overrides Units\*.slk
instead of shipping war3map.w3u. None of this is a bug in your map. 3.0.0 stopped accepting
data that older versions accepted.
This script fixes five data problems in that data.
1. The `file` column in Units\UnitUI.slk and Units\ItemData.slk -> THE crash.
Since 1.32 the art fields live in Units\*Skin.txt, and 3.0.0's own UnitUI.slk has no
`file` column at all. Older versions ignored the extra column. 3.0.0 reads the field,
gets -1, and uses it as an index. The game dies on the first unit created in the match,
with any unit, even a base game footman, and nothing in the log.
Fix: move every model path into Units\UnitSkin.txt / Units\ItemSkin.txt and drop the
column. A .txt profile is merged with the game data, a .slk replaces the whole table,
so listing only your own units is safe.
2. Numbers stored as text: walk="280.", scale="1.", shadowW="140.".
NOT a crash. This was a suspect before the `file` column turned up, and it is not the
cause: the map this came from still ships 236 of these in UnitUI.slk and loads fine on
3.0.0.
3. Incomplete button positions: `Buttonpos=,2` instead of `0,2`, `Researchbuttonpos=1`
instead of `1,0`, or the key missing entirely.
4. A stray `*/` in a .fdf with no `/*` anywhere. 3.0.0 rejects the token and closes the
game on the loading screen.
5. Ability levels above 4 read empty values. Found by arakunido on wc3edit.
An ability SLK only has per-level columns for levels 1 to 4 (DataA1..DataA4,
Cool1..Cool4, ...). Up to 2.0.4 any level above 4 used the level 4 values. 3.0.0's own
SLKs have columns for levels 5 and 6, so a map SLK that stops at 4 reads empty from
level 5 up. Every per-level field of that ability reads as empty or 0: damage values,
cooldown, range, and flags such as the Visible option of Channel abilities, which makes
hero skills vanish from the command buttons.
Fix: in Units\AbilityData.slk, add level 5 and level 6 columns to the 20 per-level
fields, each a copy of that row's level 4 value.
USAGE
python slkfix.py <map folder> # report only, changes nothing
python slkfix.py <map folder> --apply # fix in place (writes .bak files)
python slkfix.py <map folder> --apply --out <other folder>
<map folder> is the ROOT of your extracted map (the folder that contains `Units`, `UI`,
`war3map.j`, ...). You can also point it straight at the `Units` folder.
Extract and repack with any MPQ editor (Ladik's MPQ Editor, MPQMaster, ...). Keep the
original file names and (re)add the files you changed.
Python 2.7 or 3.x, no dependencies.
Public domain. Do whatever you want with it.
"""
from __future__ import print_function
import io
import os
import re
import sys
# ---------------------------------------------------------------- SLK reading and writing
def read_lines(path):
raw = io.open(path, 'rb').read().decode('utf-8', 'surrogateescape')
return raw.replace('\r\n', '\n').split('\n'), ('\r\n' in raw)
def write_lines(path, lines, crlf):
sep = '\r\n' if crlf else '\n'
io.open(path, 'wb').write(sep.join(lines).encode('utf-8', 'surrogateescape'))
def cells(lines):
"""Yield (line index, x, y, raw K value) for every C record.
X and Y are sticky in SLK: a record that omits one keeps the previous value. Getting
this wrong is the classic way to corrupt a file, so it is resolved here once.
"""
cur_x = cur_y = None
for i, line in enumerate(lines):
if not line.startswith('C;'):
continue
x = y = k = None
for field in line.split(';')[1:]:
if field[:1] == 'X' and field[1:2].isdigit():
x = int(re.match(r'X(\d+)', field).group(1))
elif field[:1] == 'Y' and field[1:2].isdigit():
y = int(re.match(r'Y(\d+)', field).group(1))
elif field[:1] == 'K':
k = field[1:]
if x is not None:
cur_x = x
if y is not None:
cur_y = y
yield i, cur_x, cur_y, k
def header(lines):
"""{column index: column name} taken from row 1."""
out = {}
for _, x, y, k in cells(lines):
if y == 1 and k:
out[x] = k.strip('"')
return out
def table(lines):
"""{row: {column name: value}} for rows 2..n."""
cols = header(lines)
rows = {}
for _, x, y, k in cells(lines):
if y is None or y < 2 or k is None:
continue
rows.setdefault(y, {})[cols.get(x, str(x))] = k.strip('"')
return rows, cols
def drop_columns(lines, crlf, drop_names):
"""Rewrite the SLK without the named columns, renumbering what is left.
Every surviving cell is written with an explicit X and Y so the result never depends on
the sticky state of the line before it.
"""
cols = header(lines)
keep = {}
n = 0
for x in sorted(cols):
if cols[x].lower() not in drop_names:
n += 1
keep[x] = n
body = []
for i, x, y, k in cells(lines):
if x not in keep or k is None:
continue
body.append((i, 'C;X%d;Y%d;K%s' % (keep[x], y, k)))
result = []
body_at = dict(body)
for i, line in enumerate(lines):
if line.startswith('C;'):
if i in body_at:
result.append(body_at[i])
else:
result.append(line)
for i, line in enumerate(result):
if line.startswith('B;'):
result[i] = re.sub(r'X\d+', 'X%d' % n, line, count=1)
break
return result, n
# ---------------------------------------------------------------- 1. models -> *Skin.txt
def read_profile(path):
"""[CODE] / key=value profile file -> {CODE: {lowercase key: value}} keeping key order."""
data, order, current = {}, [], None
if not os.path.exists(path):
return data, order
raw = io.open(path, 'rb').read().decode('utf-8', 'surrogateescape')
for line in raw.replace('\r\n', '\n').split('\n'):
s = line.strip()
if s.startswith('[') and s.endswith(']'):
current = s[1:-1]
if current not in data:
data[current] = {}
order.append(current)
elif current and '=' in s and not s.startswith('//'):
key, _, value = s.partition('=')
data[current][key.strip()] = value
return data, order
def write_profile(path, data, order, banner):
lines = list(banner) + ['']
for code in order:
lines.append('[%s]' % code)
for key in sorted(data[code]):
lines.append('%s=%s' % (key, data[code][key]))
lines.append('')
io.open(path, 'wb').write('\r\n'.join(lines).encode('utf-8', 'surrogateescape'))
def move_models(units_dir, slk_name, id_column, skin_name, apply_it, out_dir, log):
slk_path = find_file(units_dir, slk_name)
if not slk_path:
log(' %-14s not in this map, skipping' % slk_name)
return 0
lines, crlf = read_lines(slk_path)
cols = header(lines)
if 'file' not in [c.lower() for c in cols.values()]:
log(' %-14s already has no `file` column' % slk_name)
return 0
rows, _ = table(lines)
models = {}
for _, row in rows.items():
code = row.get(id_column) or row.get(id_column.lower())
path = row.get('file')
if code and path:
models[code] = path
log(' %-14s %d models in the `file` column' % (slk_name, len(models)))
if not apply_it:
return len(models)
skin_path = os.path.join(out_dir, skin_name)
data, order = read_profile(find_file(units_dir, skin_name) or skin_path)
for code in sorted(models):
if code not in data:
data[code] = {}
order.append(code)
data[code]['file'] = models[code]
write_profile(skin_path, data, order, [
'// Model paths moved out of %s by slkfix.py.' % slk_name,
'// Warcraft III 3.0.0 crashes on the `file` column in that SLK; this is where the',
'// current engine reads the model from. Profile .txt files are merged, not replaced.',
])
new_lines, kept = drop_columns(lines, crlf, set(['file']))
target = os.path.join(out_dir, os.path.basename(slk_path))
backup(slk_path, target, apply_it)
write_lines(target, new_lines, crlf)
log(' %-14s -> %s (%d entries), SLK rewritten with %d columns'
% (slk_name, skin_name, len(models), kept))
return len(models)
# ---------------------------------------------------------------- 2. numbers as text
NUMBER = re.compile(r'^-?\d+\.?\d*$')
TRAILING_DOT = re.compile(r'^(-?\d+)\.$')
def fix_numbers(units_dir, apply_it, out_dir, log):
total = 0
for name in sorted(os.listdir(units_dir)):
if not name.lower().endswith('.slk'):
continue
# step 1 may already have rewritten this file into out_dir; work on that copy, or the
# `file` column would come back
path = find_file(out_dir, name) or os.path.join(units_dir, name)
lines, crlf = read_lines(path)
cols = header(lines)
rows, _ = table(lines)
# A column is numeric when almost everything in it already is a number. That is more
# reliable than a hand-written list of column names, which never survives a map that
# invented its own.
numeric = set()
for x, col in cols.items():
values = [r.get(col) for r in rows.values() if r.get(col) not in (None, '')]
if len(values) < 4:
continue
good = sum(1 for v in values if NUMBER.match(v))
if good >= len(values) * 0.9:
numeric.add(col)
changed = 0
out = []
for i, line in enumerate(lines):
out.append(line)
for i, x, y, k in cells(lines):
if y is None or y < 2 or k is None or not k.startswith('"'):
continue
col = cols.get(x)
if col not in numeric:
continue
value = k.strip('"')
m = TRAILING_DOT.match(value)
if m:
value = m.group(1)
elif not NUMBER.match(value):
continue
out[i] = re.sub(r'K"[^"]*"', 'K' + value, out[i], count=1)
changed += 1
if changed:
total += changed
log(' %-22s %d numeric cells were quoted text' % (name, changed))
if apply_it:
target = os.path.join(out_dir, name)
backup(path, target, apply_it)
write_lines(target, out, crlf)
if not total:
log(' no numbers stored as text')
return total
# ---------------------------------------------------------------- 3. button positions
POSKEYS = ('buttonpos', 'researchbuttonpos', 'unbuttonpos')
def fix_buttonpos(units_dir, apply_it, out_dir, log):
total = 0
for name in sorted(os.listdir(units_dir)):
if not name.lower().endswith('.txt'):
continue
path = find_file(out_dir, name) or os.path.join(units_dir, name)
raw = io.open(path, 'rb').read().decode('utf-8', 'surrogateescape')
crlf = '\r\n' in raw
lines = raw.replace('\r\n', '\n').split('\n')
changed = 0
for i, line in enumerate(lines):
s = line.strip()
if '=' not in s or s.startswith('//'):
continue
key, _, value = s.partition('=')
if key.strip().lower() not in POSKEYS:
continue
value = value.strip()
if ',' not in value:
if value == '':
new = '0,0'
else:
new = value + ',0' # a lone number is the X
else:
a, _, b = value.partition(',')
new = (a.strip() or '0') + ',' + (b.strip() or '0')
if new != value:
lines[i] = key + '=' + new
changed += 1
if changed:
total += changed
log(' %-26s %d half-written button positions' % (name, changed))
if apply_it:
target = os.path.join(out_dir, name)
backup(path, target, apply_it)
io.open(target, 'wb').write(
('\r\n' if crlf else '\n').join(lines).encode('utf-8', 'surrogateescape'))
if not total:
log(' no half-written button positions')
return total
# ---------------------------------------------------------------- 4. stray */ in a .fdf
def fix_fdf(root, apply_it, log):
total = 0
for base, _, names in os.walk(root):
for name in names:
if not name.lower().endswith('.fdf'):
continue
path = os.path.join(base, name)
raw = io.open(path, 'rb').read().decode('utf-8', 'surrogateescape')
if '*/' not in raw or '/*' in raw:
continue
n = raw.count('*/')
total += n
log(' %-26s %d stray */ with no /* anywhere' % (name, n))
if apply_it:
io.open(path + '.bak', 'wb').write(raw.encode('utf-8', 'surrogateescape'))
io.open(path, 'wb').write(raw.replace('*/', '').encode('utf-8', 'surrogateescape'))
if not total:
log(' no stray comment terminators')
return total
# ---------------------------------------------------------------- 5. ability levels above 4
# The per-level fields of Units\AbilityData.slk, as listed by arakunido.
LEVEL_FIELDS = ['Area', 'BuffID', 'Cast', 'Cool', 'Cost',
'DataA', 'DataB', 'DataC', 'DataD', 'DataE', 'DataF', 'DataG', 'DataH', 'DataI',
'Dur', 'EfctID', 'HeroDur', 'Rng', 'UnitID', 'targs']
NEW_LEVELS = (5, 6)
def add_levels(units_dir, apply_it, out_dir, log):
name = 'AbilityData.slk'
path = find_file(out_dir, name) or find_file(units_dir, name)
if not path:
log(' %-22s not in this map, skipping' % name)
return 0
base = os.path.basename(path)
lines, crlf = read_lines(path)
cols = header(lines)
by_name = dict((v, x) for x, v in cols.items())
if any(('%s5' % f) in by_name for f in LEVEL_FIELDS):
log(' %-22s already has level 5 columns' % base)
return 0
# new columns go after the last one; each remembers which level 4 column it copies
last = max(cols) if cols else 0
plan = []
for f in LEVEL_FIELDS:
x4 = by_name.get('%s4' % f)
if x4 is None:
continue
new = []
for level in NEW_LEVELS:
last += 1
new.append((last, '%s%d' % (f, level)))
plan.append((x4, new))
if not plan:
log(' %-22s has no per-level columns, skipping' % base)
return 0
added = len(plan) * len(NEW_LEVELS)
log(' %-22s %d per-level fields stop at level 4, %d columns to add' % (base, len(plan), added))
if not apply_it:
return added
resolved = list(cells(lines))
level4 = {}
for _, x, y, k in resolved:
if k is not None and y is not None and y > 1:
level4[(y, x)] = k
def new_cells(y):
out = []
for x4, new in plan:
for xn, col_name in new:
if y == 1:
out.append('C;X%d;Y1;K"%s"' % (xn, col_name))
else:
k = level4.get((y, x4))
if k is not None:
out.append('C;X%d;Y%d;K%s' % (xn, y, k))
return out
# every cell is rewritten with an explicit X and Y, and each row gets its new cells right
# after its last one, so nothing depends on the sticky state of the record before it
at = dict((i, (x, y, k)) for i, x, y, k in resolved)
result = []
row = None
for i, line in enumerate(lines):
if i in at:
x, y, k = at[i]
if row is not None and y != row:
result.extend(new_cells(row))
row = y
if k is not None:
result.append('C;X%d;Y%d;K%s' % (x, y, k))
else:
if line.startswith('E') and row is not None:
result.extend(new_cells(row))
row = None
result.append(line)
if row is not None:
result.extend(new_cells(row))
for i, line in enumerate(result):
if line.startswith('B;'):
result[i] = re.sub(r'X\d+', 'X%d' % last, line, count=1)
break
target = os.path.join(out_dir, base)
backup(path, target, apply_it)
write_lines(target, result, crlf)
log(' %-22s %d columns added as copies of level 4, %d columns now' % (base, added, last))
return added
# ---------------------------------------------------------------- helpers
def find_file(folder, name):
"""Case-insensitive lookup: MPQ contents come out in whatever case the author used."""
if not os.path.isdir(folder):
return None
for entry in os.listdir(folder):
if entry.lower() == name.lower():
return os.path.join(folder, entry)
return None
IN_PLACE = True
def backup(source, target, apply_it):
"""Keep a .bak only when we are overwriting the map's own file."""
if not (apply_it and IN_PLACE):
return
if os.path.abspath(source) == os.path.abspath(target) and not os.path.exists(source + '.bak'):
io.open(source + '.bak', 'wb').write(io.open(source, 'rb').read())
def main():
args = [a for a in sys.argv[1:] if not a.startswith('--')]
apply_it = '--apply' in sys.argv
out_dir = None
for a in sys.argv[1:]:
if a.startswith('--out='):
out_dir = a[6:]
if '--out' in sys.argv:
out_dir = sys.argv[sys.argv.index('--out') + 1]
if not args:
print(__doc__)
return 1
root = args[0]
units = root if os.path.basename(root.rstrip('\\/')).lower() == 'units' \
else (find_file(root, 'Units') or root)
if not os.path.isdir(units):
print('could not find the Units folder under %r' % root)
return 1
global IN_PLACE
out = out_dir or units
IN_PLACE = os.path.abspath(out) == os.path.abspath(units)
if apply_it and not IN_PLACE:
# a separate output folder starts as a copy, so what comes out is a complete Units
# folder you can drop straight back into the MPQ
if not os.path.isdir(out):
os.makedirs(out)
for entry in os.listdir(units):
src = os.path.join(units, entry)
if os.path.isfile(src):
io.open(os.path.join(out, entry), 'wb').write(io.open(src, 'rb').read())
def log(msg):
print(msg)
print('map root : %s' % os.path.abspath(root))
print('Units : %s' % os.path.abspath(units))
print('mode : %s' % ('APPLYING CHANGES' if apply_it else 'report only (add --apply to write)'))
print('')
print('1. model paths in the `file` column (this is the one that crashes 3.0.0)')
n1 = move_models(units, 'UnitUI.slk', 'unitUIID', 'UnitSkin.txt', apply_it, out, log)
n1 += move_models(units, 'ItemData.slk', 'itemID', 'ItemSkin.txt', apply_it, out, log)
print('')
print('2. numbers stored as text (cleanup, not a crash)')
n2 = fix_numbers(units, apply_it, out, log)
print('')
print('3. half-written button positions')
n3 = fix_buttonpos(units, apply_it, out, log)
print('')
print('4. stray */ in .fdf files')
n4 = fix_fdf(root, apply_it, log)
print('')
print('5. ability levels above 4 (found by arakunido)')
n5 = add_levels(units, apply_it, out, log)
print('')
print('models moved %d | numbers %d | button positions %d | fdf %d | level columns %d'
% (n1, n2, n3, n4, n5))
if not apply_it and (n1 or n2 or n3 or n4 or n5):
print('nothing was written -- run again with --apply')
return 0
if __name__ == '__main__':
sys.exit(main())
