#target Photoshop

function checkSelection() {
    var flag = true;
    try {
        var desc = new ActionDescriptor();
        var ref = new ActionReference();
        ref.putProperty(charIDToTypeID("Chnl"), charIDToTypeID("fsel"));
        desc.putReference(charIDToTypeID("null"), ref);
        executeAction(charIDToTypeID("Trnf"), desc, DialogModes.NO);
    } catch (e) {
        flag = false;
    }
    return flag;
}

if (checkSelection()) {
    alert("選択範囲があります。");
} else {
    alert("選択範囲がありません。");
}